Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.27188736 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.019781662 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.019776382 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1193 1194 1195 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.648839376Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.701150835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.702025758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.702133317Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.702154097Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.702172537Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198} {"msg":"Crucible 79dd5dbc-3d2b-4101-a3c0-7e59c147f1c3 has session id: 30db7721-9ac3-4636-b61a-bf5b750861db","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.702951401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Upstairs opts: Upstairs UUID: 79dd5dbc-3d2b-4101-a3c0-7e59c147f1c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.7030863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Crucible stats registered with UUID: 79dd5dbc-3d2b-4101-a3c0-7e59c147f1c3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70311268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:20.70312931Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704089462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704132492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198} {"msg":"79dd5dbc-3d2b-4101-a3c0-7e59c147f1c3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704198972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704225661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"0","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704308741Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"0","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704327521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"1","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704342091Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"1","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70438437Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"2","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70441387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"2","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70444293Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70446852Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70449331Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"ds_connection connected from Ok(127.0.0.1:62952)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704693268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"ds_connection connected from Ok(127.0.0.1:43598)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704722678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"ds_connection connected from Ok(127.0.0.1:36836)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.704737998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.705897779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"1","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.705937219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"0","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.706037208Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"2","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.707148819Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"2","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.707176299Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"0","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.707192059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"1","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.707929654Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708037593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708253791Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708276591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708294111Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708310981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708327341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70834461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70836145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70837817Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.7084155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70843159Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70844738Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70846319Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"":"downstairs","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708479049Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.70849394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"0","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708510149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"1","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708526209Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"client":"2","":"downstairs","session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708542129Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708557229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"79dd5dbc-3d2b-4101-a3c0-7e59c147f1c3 is now active with session: 30db7721-9ac3-4636-b61a-bf5b750861db","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708573019Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T04:59:20.708587739Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198,"session_id":"30db7721-9ac3-4636-b61a-bf5b750861db"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708607109Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198} {"msg":"Activated sub_volume 79dd5dbc-3d2b-4101-a3c0-7e59c147f1c3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:20.708620839Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.826443132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:24.869702639Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.870086916Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.901279742Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.901816538Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.901858428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.901876527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.901894357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} {"msg":"Crucible 0eee8998-530a-410e-b0cd-a01dc54554e1 has session id: ca300396-17b1-453e-b23e-6ecdc46d0fd5","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.902431233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Upstairs opts: Upstairs UUID: 0eee8998-530a-410e-b0cd-a01dc54554e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.902518853Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Crucible stats registered with UUID: 0eee8998-530a-410e-b0cd-a01dc54554e1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.902544542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:24.902559213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.902989469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903029119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} {"msg":"0eee8998-530a-410e-b0cd-a01dc54554e1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903092209Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903109538Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"0","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903138408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"0","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903153258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"1","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903167278Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"1","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903180288Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"2","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903204048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"2","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903230247Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903249327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903264207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58103)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903295207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46869)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903599295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35505)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.903638624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.904093701Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"2","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.904122571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"1","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.90414565Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"0","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.904741236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"2","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.904902025Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"1","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.904922215Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"0","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905335272Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.90549292Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905610579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905631969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905646799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905660399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905673839Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905691129Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905704799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905717909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905731619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905744169Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905756949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905769678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905782248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"":"downstairs","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905795418Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905807448Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"0","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905821678Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"1","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905843768Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"client":"2","":"downstairs","session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905857078Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905869248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"0eee8998-530a-410e-b0cd-a01dc54554e1 is now active with session: ca300396-17b1-453e-b23e-6ecdc46d0fd5","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905881708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905895668Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200,"session_id":"ca300396-17b1-453e-b23e-6ecdc46d0fd5"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905907777Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} {"msg":"Activated sub_volume 0eee8998-530a-410e-b0cd-a01dc54554e1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:24.905918977Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 148 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 376 len 16384 05/30 Write block 378 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Write block 19 len 20480 data: 2 2 2 2 2 07/30 Read block 427 len 28672 08/30 Write block 576 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Write block 33 len 36864 data: 2 2 2 2 2 2 2 2 2 10/30 Read block 100 len 16384 11/30 Flush 12/30 Write block 481 len 24576 data: 2 2 2 2 2 2 13/30 Write block 518 len 20480 data: 2 2 2 2 2 14/30 Read block 577 len 20480 15/30 Read block 38 len 36864 16/30 Read block 273 len 20480 17/30 Write block 367 len 24576 data: 2 2 2 2 2 2 18/30 Write block 158 len 24576 data: 2 2 2 2 2 2 19/30 Write block 284 len 16384 data: 2 2 2 2 20/30 Flush 21/30 Flush 22/30 Write block 507 len 20480 data: 2 2 2 2 2 23/30 Write block 117 len 16384 data: 2 2 2 2 24/30 Write block 205 len 24576 data: 2 2 2 2 2 2 25/30 Write block 400 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Read block 494 len 20480 27/30 Read block 23 len 36864 28/30 Read block 255 len 32768 29/30 Write block 459 len 20480 data: 2 2 2 2 2 30/30 Read block 446 len 36864 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 9 7 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:24.927343787Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1200} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 T T T 7 140-159 2 2 2 4 4 4 F F F 8 160-179 2 2 2 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 T T T 24 480-499 2 2 2 4 4 4 F F F 25 500-519 2 2 2 4 4 4 T T T 26 520-539 2 2 2 4 4 4 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 2, Max flush: 4 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 2 2 2 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 3 3 3 5 5 5 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 2 2 2 4 4 4 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 2 2 2 4 4 4 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.209067594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:29.251519747Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.251877884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.282700883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.283312509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.283347038Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.283360798Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.283374908Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} {"msg":"Crucible bfb0cbee-205d-48db-9010-6e6bb3c16d44 has session id: 30416aff-5aa6-45a8-83f9-eb2204f003ed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.283848294Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Upstairs opts: Upstairs UUID: bfb0cbee-205d-48db-9010-6e6bb3c16d44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.283891964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Crucible stats registered with UUID: bfb0cbee-205d-48db-9010-6e6bb3c16d44","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.283917624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:29.283939104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284344451Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.28436959Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} {"msg":"bfb0cbee-205d-48db-9010-6e6bb3c16d44 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.28443638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.28445425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"0","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.2844811Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"0","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.28449548Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"1","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284524919Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"1","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284552839Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"2","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284580419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"2","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284607509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284632188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284657198Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:51758)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284682998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:55441)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284708758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:35413)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.284829037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.285327533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"1","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.285356213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"2","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.285492132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"0","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.285970529Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"2","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.285999698Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"1","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286021608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"0","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286564784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286588864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[0]R flush_numbers[0..12]: [3, 3, 3, 1, 1, 5, 5, 4, 4, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286628693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[0]R generation[0..12]: [2, 2, 2, 1, 1, 3, 3, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286645624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286660123Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[1]R flush_numbers[0..12]: [3, 3, 3, 1, 1, 5, 5, 4, 4, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286673863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[1]R generation[0..12]: [2, 2, 2, 1, 1, 3, 3, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286687253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286701033Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[2]R flush_numbers[0..12]: [3, 3, 3, 1, 1, 5, 5, 4, 4, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286714263Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[2]R generation[0..12]: [2, 2, 2, 1, 1, 3, 3, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286727403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286741183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286753803Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286766583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286779112Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286795552Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"":"downstairs","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286809372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286821472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"0","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286834902Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"1","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286854732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"client":"2","":"downstairs","session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286868092Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286880232Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"bfb0cbee-205d-48db-9010-6e6bb3c16d44 is now active with session: 30416aff-5aa6-45a8-83f9-eb2204f003ed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286892902Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286904871Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208,"session_id":"30416aff-5aa6-45a8-83f9-eb2204f003ed"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286916982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} {"msg":"Activated sub_volume bfb0cbee-205d-48db-9010-6e6bb3c16d44","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:29.286927981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 482 len 32768 data: 3 3 3 3 3 2 2 2 02/30 Flush 03/30 Write block 116 len 36864 data: 2 3 3 3 3 2 2 2 2 04/30 Write block 539 len 16384 data: 2 2 2 2 05/30 Read block 4 len 24576 06/30 Read block 274 len 16384 07/30 Write block 324 len 16384 data: 2 2 2 2 08/30 Read block 407 len 36864 09/30 Write block 396 len 24576 data: 2 2 2 2 3 3 10/30 Read block 251 len 24576 11/30 Flush 12/30 Write block 435 len 24576 data: 2 2 2 2 2 2 13/30 Write block 97 len 4096 data: 2 14/30 Write block 408 len 24576 data: 3 3 2 2 2 2 15/30 Read block 277 len 36864 16/30 Flush 17/30 Write block 595 len 16384 data: 2 2 2 2 18/30 Read block 212 len 12288 19/30 Write block 333 len 20480 data: 2 2 2 2 2 20/30 Read block 152 len 28672 21/30 Read block 209 len 4096 22/30 Write block 315 len 8192 data: 2 2 23/30 Write block 26 len 12288 data: 2 2 2 24/30 Read block 426 len 12288 25/30 Write block 14 len 4096 data: 2 26/30 Write block 0 len 24576 data: 2 2 2 2 2 2 27/30 Read block 373 len 24576 28/30 Write block 9 len 40960 data: 2 2 2 2 2 3 2 2 2 2 29/30 Write block 521 len 4096 data: 3 30/30 Read block 315 len 36864 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 14 14 13 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:29.306650964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 T T T 1 020-039 2 2 2 3 3 3 T T T 2 040-059 2 2 2 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 8 8 8 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 2 2 2 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 4 4 4 7 7 7 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 4 4 4 7 7 7 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 4 4 4 8 8 8 F F F 22 440-459 4 4 4 8 8 8 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 4 4 4 7 7 7 T T T 27 540-559 4 4 4 7 7 7 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 2 2 2 3 3 3 T T T Max gen: 4, Max flush: 8 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 9 9 9 F F F 1 020-039 5 5 5 9 9 9 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 8 8 8 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 2 2 2 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 5 5 5 9 9 9 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 4 4 4 F F F 19 380-399 4 4 4 7 7 7 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 4 4 4 8 8 8 F F F 22 440-459 4 4 4 8 8 8 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 5 5 5 9 9 9 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 5 5 5 9 9 9 F F F Max gen: 5, Max flush: 9 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.582794654Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.626807114Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.627557228Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.627610238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.627629548Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.627647608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} {"msg":"Crucible ac53665a-8419-4c68-9bea-4259e9f70773 has session id: 3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628170324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Upstairs opts: Upstairs UUID: ac53665a-8419-4c68-9bea-4259e9f70773, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628210174Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Crucible stats registered with UUID: ac53665a-8419-4c68-9bea-4259e9f70773","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628233763Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:33.628259773Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628798449Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628832219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} {"msg":"ac53665a-8419-4c68-9bea-4259e9f70773 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628877849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628894969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"0","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628920418Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"0","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628934398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"1","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628948508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"1","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628961438Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"2","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.628977868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"2","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.629019148Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.629046728Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.629072807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:61647)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.629269146Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:48298)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.629312766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:61075)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.629340325Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.62997259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"0","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.63001101Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"1","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.63003859Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"2","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.630537986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"2","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.630671955Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"1","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.630794434Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"0","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631147582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631270851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[0]R flush_numbers[0..12]: [9, 9, 3, 1, 8, 7, 7, 4, 4, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.63133457Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[0]R generation[0..12]: [5, 5, 2, 1, 4, 4, 4, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.63135472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.6313693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[1]R flush_numbers[0..12]: [9, 9, 3, 1, 8, 7, 7, 4, 4, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.6313828Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[1]R generation[0..12]: [5, 5, 2, 1, 4, 4, 4, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.63139596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.6314096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[2]R flush_numbers[0..12]: [9, 9, 3, 1, 8, 7, 7, 4, 4, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.63142708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[2]R generation[0..12]: [5, 5, 2, 1, 4, 4, 4, 2, 2, 1, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.63144075Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631454649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631468179Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631481459Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631494209Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631507019Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"":"downstairs","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631519989Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631531709Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid"Wait for a query_work_queue command to finish before sending IO :1216,"client":"0","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631552029Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"1","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631564949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"client":"2","":"downstairs","session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631577849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631589988Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"ac53665a-8419-4c68-9bea-4259e9f70773 is now active with session: 3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631602558Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631614228Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216,"session_id":"3d6bbef4-d8f4-44c8-8b7e-1dbf04a2c1ea"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631628858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} {"msg":"Activated sub_volume ac53665a-8419-4c68-9bea-4259e9f70773","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:33.631640118Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 429 len 28672 data: 2 2 2 2 2 2 3 02/30 Read block 238 len 32768 03/30 Write block 7 len 36864 data: 2 2 3 3 3 3 3 4 3 04/30 Read block 11 len 32768 05/30 Write block 55 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Read block 239 len 12288 07/30 Read block 85 len 8192 08/30 Read block 426 len 12288 09/30 Read block 31 len 4096 10/30 Read block 428 len 28672 11/30 Write block 370 len 40960 data: 3 3 3 2 2 2 2 2 3 3 12/30 Write block 385 len 20480 data: 3 3 2 2 2 13/30 Write block 338 len 16384 data: 2 2 2 2 14/30 Write block 178 len 20480 data: 2 2 2 2 2 15/30 Read block 116 len 24576 16/30 Read block 552 len 32768 17/30 Read block 387 len 36864 18/30 Flush 19/30 Write block 112 len 28672 data: 2 2 2 2 3 4 4 20/30 Write block 73 len 32768 data: 2 2 2 2 2 2 2 2 21/30 Write block 72 len 16384 data: 2 3 3 3 22/30 Read block 484 len 24576 23/30 Read block 426 len 24576 24/30 Write block 537 len 8192 data: 2 2 25/30 Write block 90 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 548 len 16384 27/30 Write block 84 len 28672 data: 2 2 2 2 2 2 3 28/30 Write block 568 len 24576 data: 2 2 2 2 2 2 29/30 Write block 460 len 16384 data: 3 3 3 3 30/30 Read block 454 len 28672 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 12 11 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:33.65269825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 5 5 5 9 9 9 F F F 2 040-059 6 6 6 10 10 10 F F F 3 060-079 6 6 6 10 10 10 T T T 4 080-099 4 4 4 8 8 8 T T T 5 100-119 4 4 4 7 7 7 T T T 6 120-139 4 4 4 7 7 7 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 6 6 6 10 10 10 F F F 19 380-399 6 6 6 10 10 10 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 6 6 6 10 10 10 F F F 22 440-459 4 4 4 8 8 8 F F F 23 460-479 3 3 3 5 5 5 T T T 24 480-499 4 4 4 6 6 6 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 5 5 5 9 9 9 T T T 27 540-559 4 4 4 7 7 7 F F F 28 560-579 2 2 2 3 3 3 T T T 29 580-599 5 5 5 9 9 9 F F F Max gen: 6, Max flush: 10 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 10 10 10 F F F 1 020-039 5 5 5 9 9 9 F F F 2 040-059 6 6 6 10 10 10 F F F 3 060-079 7 7 7 11 11 11 F F F 4 080-099 7 7 7 11 11 11 F F F 5 100-119 7 7 7 11 11 11 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 2 2 2 4 4 4 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 4 4 4 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 6 6 6 10 10 10 F F F 19 380-399 6 6 6 10 10 10 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 6 6 6 10 10 10 F F F 22 440-459 4 4 4 8 8 8 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 4 4 4 7 7 7 F F F 28 560-579 7 7 7 11 11 11 F F F 29 580-599 5 5 5 9 9 9 F F F Max gen: 7, Max flush: 11 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.893986199Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.938712724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.9393372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.93939037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.939418019Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.939443079Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} {"msg":"Crucible b9a9326d-cccb-44a3-b2c7-413df9ff6808 has session id: 3bd2e221-5c08-4173-a682-5c40c0d48115","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940080274Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Upstairs opts: Upstairs UUID: b9a9326d-cccb-44a3-b2c7-413df9ff6808, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940115034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Crucible stats registered with UUID: b9a9326d-cccb-44a3-b2c7-413df9ff6808","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940130304Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:37.940143134Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940520721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940553491Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} {"msg":"b9a9326d-cccb-44a3-b2c7-413df9ff6808 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940609Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94063617Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"0","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94066418Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"0","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94067872Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"1","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94069713Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"1","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940722629Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"2","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940749789Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"2","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940771259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940785559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940799029Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"ds_connection connected from Ok(127.0.0.1:58962)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940821059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"ds_connection connected from Ok(127.0.0.1:47075)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940835909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"ds_connection connected from Ok(127.0.0.1:63988)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.940866508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.941593353Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"1","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.941633883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"0","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.941670622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"2","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.942258668Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"1","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.942288278Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"2","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.942317548Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"0","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.942913583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.942949403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[0]R flush_numbers[0..12]: [10, 9, 10, 11, 11, 11, 7, 4, 10, 10, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.942978143Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[0]R generation[0..12]: [6, 5, 6, 7, 7, 7, 4, 2, 6, 6, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.942996552Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943021982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[1]R flush_numbers[0..12]: [10, 9, 10, 11, 11, 11, 7, 4, 10, 10, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943037682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[1]R generation[0..12]: [6, 5, 6, 7, 7, 7, 4, 2, 6, 6, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943051072Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943065132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[2]R flush_numbers[0..12]: [10, 9, 10, 11, 11, 11, 7, 4, 10, 10, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943078582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[2]R generation[0..12]: [6, 5, 6, 7, 7, 7, 4, 2, 6, 6, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943099352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943125571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943149581Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T04:59:37.943165381Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943182421Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943195081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"":"downstairs","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943208031Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943219891Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"0","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943236261Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"1","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943249441Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"client":"2","":"downstairs","session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94326225Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.943274441Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"b9a9326d-cccb-44a3-b2c7-413df9ff6808 is now active with session: 3bd2e221-5c08-4173-a682-5c40c0d48115","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94328717Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94329906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224,"session_id":"3bd2e221-5c08-4173-a682-5c40c0d48115"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94331119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} {"msg":"Activated sub_volume b9a9326d-cccb-44a3-b2c7-413df9ff6808","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:37.94332228Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 545 len 24576 data: 2 2 2 2 2 2 02/30 Write block 557 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Write block 552 len 4096 data: 2 04/30 Flush 05/30 Write block 506 len 4096 data: 2 06/30 Flush 07/30 Write block 130 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Write block 400 len 40960 data: 4 4 3 3 3 3 3 3 4 4 09/30 Flush 10/30 Write block 292 len 8192 data: 2 2 11/30 Read block 281 len 16384 12/30 Read block 509 len 8192 13/30 Write block 23 len 16384 data: 3 2 2 3 14/30 Read block 508 len 12288 15/30 Read block 8 len 8192 16/30 Read block 133 len 16384 17/30 Write block 131 len 32768 data: 3 3 3 3 3 3 3 2 18/30 Write block 124 len 8192 data: 3 2 19/30 Write block 3 len 20480 data: 3 3 3 2 3 20/30 Read block 336 len 12288 21/30 Write block 405 len 16384 data: 4 4 4 5 22/30 Read block 268 len 32768 23/30 Read block 561 len 8192 24/30 Flush 25/30 Write block 154 len 8192 data: 3 2 26/30 Flush 27/30 Write block 243 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Write block 431 len 8192 data: 3 3 29/30 Write block 419 len 28672 data: 2 2 2 2 2 2 2 30/30 Write block 378 len 20480 data: 4 4 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 2 Sent Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 3 8 Done 1 2 1 4 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:37.960851619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 6 6 6 10 10 10 F F F 3 060-079 7 7 7 11 11 11 F F F 4 080-099 7 7 7 11 11 11 F F F 5 100-119 7 7 7 11 11 11 F F F 6 120-139 8 8 8 15 15 15 F F F 7 140-159 8 8 8 16 16 16 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 T T T 13 260-279 2 2 2 2 2 2 F F F 14 280-299 8 8 8 15 15 15 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 6 6 6 10 10 10 T T T 19 380-399 6 6 6 10 10 10 T T T 20 400-419 8 8 8 15 15 15 T T T 21 420-439 6 6 6 10 10 10 T T T 22 440-459 4 4 4 8 8 8 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 8 8 8 13 13 13 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 12 12 12 F F F 29 580-599 5 5 5 9 9 9 F F F Max gen: 8, Max flush: 16 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 6 6 6 10 10 10 F F F 3 060-079 7 7 7 11 11 11 F F F 4 080-099 7 7 7 11 11 11 F F F 5 100-119 7 7 7 11 11 11 F F F 6 120-139 8 8 8 15 15 15 F F F 7 140-159 8 8 8 16 16 16 F F F 8 160-179 6 6 6 10 10 10 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 9 9 9 17 17 17 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 8 8 8 15 15 15 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 9 9 9 17 17 17 F F F 21 420-439 9 9 9 17 17 17 F F F 22 440-459 4 4 4 8 8 8 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 8 8 8 13 13 13 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 8 8 8 12 12 12 F F F 28 560-579 8 8 8 12 12 12 F F F 29 580-599 5 5 5 9 9 9 F F F Max gen: 9, Max flush: 17 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.19800167Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:42.240299143Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.240654861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.271604109Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.272137245Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.272179145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.272197334Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.272216264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} {"msg":"Crucible 765581d5-ddcb-4e74-affe-cea8a209005a has session id: 988c57d1-c0ec-4a65-9161-108b58058e6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.272720821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Upstairs opts: Upstairs UUID: 765581d5-ddcb-4e74-affe-cea8a209005a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.27275985Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Crucible stats registered with UUID: 765581d5-ddcb-4e74-affe-cea8a209005a","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.27278032Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:42.27279338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273268966Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273302296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} {"msg":"765581d5-ddcb-4e74-affe-cea8a209005a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273428165Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273467475Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"0","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273512035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"0","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273541314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"1","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273570454Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"1","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273592664Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"2","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273619044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"2","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273643384Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273665354Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273686613Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34269)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273701383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44505)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273853532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48182)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.273898592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.274391748Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"1","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.274420128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"2","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.274542787Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"0","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275131372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"1","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275183132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"2","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275217232Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"0","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275795998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275830857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[0]R flush_numbers[0..12]: [15, 15, 10, 11, 11, 11, 15, 16, 10, 10, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275886037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[0]R generation[0..12]: [8, 8, 6, 7, 7, 7, 8, 8, 6, 6, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275915077Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275931366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[1]R flush_numbers[0..12]: [15, 15, 10, 11, 11, 11, 15, 16, 10, 10, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275945266Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[1]R generation[0..12]: [8, 8, 6, 7, 7, 7, 8, 8, 6, 6, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275963986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.275987296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[2]R flush_numbers[0..12]: [15, 15, 10, 11, 11, 11, 15, 16, 10, 10, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276007566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[2]R generation[0..12]: [8, 8, 6, 7, 7, 7, 8, 8, 6, 6, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276030626Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276053856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276082725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276109185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276134865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276150045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"":"downstairs","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276168595Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Transition from WaitQuorum to ActiveWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276197015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"0","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276223804Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"1","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276243094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"client":"2","":"downstairs","session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276256524Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276272244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"765581d5-ddcb-4e74-affe-cea8a209005a is now active with session: 988c57d1-c0ec-4a65-9161-108b58058e6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276295544Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276318754Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232,"session_id":"988c57d1-c0ec-4a65-9161-108b58058e6d"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276340514Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} {"msg":"Activated sub_volume 765581d5-ddcb-4e74-affe-cea8a209005a","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:42.276352603Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 505 len 24576 data: 2 3 3 3 3 3 02/30 Read block 234 len 12288 03/30 Flush 04/30 Write block 547 len 4096 data: 3 05/30 Write block 588 len 16384 data: 2 2 2 2 06/30 Read block 599 len 4096 07/30 Flush 08/30 Write block 287 len 32768 data: 3 2 2 2 2 3 3 2 09/30 Read block 270 len 32768 10/30 Read block 91 len 16384 11/30 Write block 232 len 4096 data: 2 12/30 Write block 55 len 40960 data: 3 3 3 3 3 3 3 3 3 2 13/30 Read block 82 len 28672 14/30 Read block 482 len 40960 15/30 Read block 58 len 36864 16/30 Read block 12 len 16384 17/30 Write block 230 len 4096 data: 2 18/30 Write block 89 len 8192 data: 3 4 19/30 Read block 72 len 12288 20/30 Write block 175 len 12288 data: 2 2 2 21/30 Write block 131 len 16384 data: 4 4 4 4 22/30 Flush 23/30 Write block 44 len 16384 data: 2 2 2 2 24/30 Read block 358 len 8192 25/30 Write block 214 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Read block 393 len 40960 27/30 Write block 151 len 32768 data: 3 3 3 4 3 2 2 3 28/30 Read block 423 len 16384 29/30 Read block 575 len 4096 30/30 Read block 202 len 36864 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:42.297009589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 10 10 10 20 20 20 T T T 3 060-079 10 10 10 20 20 20 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 7 7 7 11 11 11 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 8 8 8 16 16 16 T T T 8 160-179 10 10 10 20 20 20 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 3 3 3 5 5 5 T T T 11 220-239 10 10 10 20 20 20 T T T 12 240-259 9 9 9 17 17 17 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 10 10 10 20 20 20 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 9 9 9 17 17 17 F F F 21 420-439 9 9 9 17 17 17 F F F 22 440-459 4 4 4 8 8 8 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 10 10 10 18 18 18 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 10 10 10 19 19 19 F F F 28 560-579 8 8 8 12 12 12 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 10, Max flush: 20 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 11 11 11 21 21 21 F F F 3 060-079 10 10 10 20 20 20 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 7 7 7 11 11 11 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 10 10 10 20 20 20 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 11 11 11 21 21 21 F F F 11 220-239 11 11 11 21 21 21 F F F 12 240-259 9 9 9 17 17 17 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 10 10 10 20 20 20 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 6 6 6 10 10 10 F F F 17 340-359 6 6 6 10 10 10 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 9 9 9 17 17 17 F F F 21 420-439 9 9 9 17 17 17 F F F 22 440-459 4 4 4 8 8 8 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 10 10 10 18 18 18 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 10 10 10 19 19 19 F F F 28 560-579 8 8 8 12 12 12 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 11, Max flush: 21 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.557874662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.601690704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60226388Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60230875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60232346Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.602337549Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} {"msg":"Crucible b8534d45-4758-4de0-ab17-809eedd56f79 has session id: 15a87e6f-f2bb-4b66-9991-7f6afb2776a4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.602868756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Upstairs opts: Upstairs UUID: b8534d45-4758-4de0-ab17-809eedd56f79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.602904685Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Crucible stats registered with UUID: b8534d45-4758-4de0-ab17-809eedd56f79","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.602924705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:46.602937795Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603403492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603428301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} {"msg":"b8534d45-4758-4de0-ab17-809eedd56f79 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603495991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603514411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"0","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60354632Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"0","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60357204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"1","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60359943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"1","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60362751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"2","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.60365335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"2","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603679329Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603703189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603728139Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:46511)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603755589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43997)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603789258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49570)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.603821738Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.604438164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"1","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.604604403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"0","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.605068989Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"1","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.605194578Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"0","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.605505356Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"2","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.605678164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.605768364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606048062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"2","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[0]R flush_numbers[0..12]: [15, 15, 21, 20, 20, 11, 20, 21, 20, 10, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606553178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[0]R generation[0..12]: [8, 8, 11, 10, 10, 7, 10, 11, 10, 6, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606574998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606589937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[1]R flush_numbers[0..12]: [15, 15, 21, 20, 20, 11, 20, 21, 20, 10, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606604137Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[1]R generation[0..12]: [8, 8, 11, 10, 10, 7, 10, 11, 10, 6, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606621987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606636357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[2]R flush_numbers[0..12]: [15, 15, 21, 20, 20, 11, 20, 21, 20, 10, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606649697Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[2]R generation[0..12]: [8, 8, 11, 10, 10, 7, 10, 11, 10, 6, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606662897Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606676717Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606689397Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606702397Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606714837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T04:59:46.606727536Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"":"downstairs","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606745687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606757536Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"0","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606770796Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"1","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606783376Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"client":"2","":"downstairs","session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606796256Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606819866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"b8534d45-4758-4de0-ab17-809eedd56f79 is now active with session: 15a87e6f-f2bb-4b66-9991-7f6afb2776a4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606836216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606848616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240,"session_id":"15a87e6f-f2bb-4b66-9991-7f6afb2776a4"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606860835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} {"msg":"Activated sub_volume b8534d45-4758-4de0-ab17-809eedd56f79","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:46.606871975Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 1 len 8192 data: 3 3 02/30 Flush 03/30 Write block 326 len 28672 data: 3 3 2 2 2 2 2 04/30 Write block 116 len 16384 data: 4 5 5 4 05/30 Read block 372 len 24576 06/30 Read block 442 len 16384 07/30 Flush 08/30 Write block 36 len 16384 data: 3 3 3 3 09/30 Read block 386 len 4096 10/30 Read block 252 len 40960 11/30 Write block 516 len 16384 data: 2 2 3 3 12/30 Flush 13/30 Write block 53 len 16384 data: 2 2 4 4 14/30 Write block 216 len 16384 data: 3 3 3 3 15/30 Write block 419 len 36864 data: 3 3 3 3 3 3 3 2 2 16/30 Write block 273 len 12288 data: 2 2 2 17/30 Read block 235 len 20480 18/30 Read block 479 len 28672 19/30 Write block 562 len 36864 data: 3 3 3 3 2 2 3 3 3 20/30 Read block 210 len 8192 21/30 Flush 22/30 Flush 23/30 Write block 347 len 4096 data: 2 24/30 Write block 449 len 36864 data: 2 2 2 2 2 2 2 2 2 25/30 Read block 68 len 32768 26/30 Read block 47 len 32768 27/30 Write block 437 len 28672 data: 3 3 3 3 2 2 2 28/30 Read block 329 len 4096 29/30 Write block 227 len 28672 data: 2 2 2 3 2 3 2 30/30 Read block 204 len 16384 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 8 8 6 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:46.626768866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 12 12 12 24 24 24 F F F 2 040-059 12 12 12 25 25 25 F F F 3 060-079 10 10 10 20 20 20 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 10 10 10 20 20 20 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 11 11 11 21 21 21 T T T 12 240-259 9 9 9 17 17 17 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 10 10 10 20 20 20 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 12 12 12 23 23 23 F F F 17 340-359 6 6 6 10 10 10 T T T 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 12 12 12 25 25 25 F F F 21 420-439 12 12 12 25 25 25 T T T 22 440-459 4 4 4 8 8 8 T T T 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 12 12 12 24 24 24 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 10 10 10 19 19 19 F F F 28 560-579 12 12 12 25 25 25 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 12, Max flush: 25 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 12 12 12 24 24 24 F F F 2 040-059 12 12 12 25 25 25 F F F 3 060-079 10 10 10 20 20 20 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 10 10 10 20 20 20 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 13 13 13 26 26 26 F F F 12 240-259 9 9 9 17 17 17 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 10 10 10 20 20 20 F F F 15 300-319 5 5 5 9 9 9 F F F 16 320-339 12 12 12 23 23 23 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 12 12 12 25 25 25 F F F 21 420-439 13 13 13 26 26 26 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 12 12 12 24 24 24 F F F 26 520-539 7 7 7 11 11 11 F F F 27 540-559 10 10 10 19 19 19 F F F 28 560-579 12 12 12 25 25 25 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 13, Max flush: 26 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.857758185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.90252206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.903151505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.903194875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.903215084Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.903230774Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} {"msg":"Crucible 1858ce63-b815-407a-ab81-a116fd553ccf has session id: 791b3f05-e662-4811-b509-d772adfe51be","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.903641301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Upstairs opts: Upstairs UUID: 1858ce63-b815-407a-ab81-a116fd553ccf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.90376788Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Crucible stats registered with UUID: 1858ce63-b815-407a-ab81-a116fd553ccf","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.90379037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:50.90380406Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904360576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904385336Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} {"msg":"1858ce63-b815-407a-ab81-a116fd553ccf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904456645Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904474465Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"0","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904501865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"0","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904525465Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"1","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904553034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"1","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904580514Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"2","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904612444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"2","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904642424Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904670424Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904694943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"ds_connection connected from Ok(127.0.0.1:34170)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904719843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"ds_connection connected from Ok(127.0.0.1:61101)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904741303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"ds_connection connected from Ok(127.0.0.1:53606)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.904765213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.905399638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"0","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.905441108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"1","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.905471198Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"2","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906023113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"1","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906060423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"0","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906077903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"2","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906636889Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906676558Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[0]R flush_numbers[0..12]: [22, 24, 25, 20, 20, 23, 20, 21, 20, 10, 25, 26]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906734978Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[0]R generation[0..12]: [12, 12, 12, 10, 10, 12, 10, 11, 10, 6, 12, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906763128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906779508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[1]R flush_numbers[0..12]: [22, 24, 25, 20, 20, 23, 20, 21, 20, 10, 25, 26]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906793457Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[1]R generation[0..12]: [12, 12, 12, 10, 10, 12, 10, 11, 10, 6, 12, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906807048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906821627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[2]R flush_numbers[0..12]: [22, 24, 25, 20, 20, 23, 20, 21, 20, 10, 25, 26]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906835627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[2]R generation[0..12]: [12, 12, 12, 10, 10, 12, 10, 11, 10, 6, 12, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906849267Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906863157Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906875947Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906888987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906901687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906914427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"":"downstairs","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906927276Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906948676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"0","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906964216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"1","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.906977216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"client":"2","":"downstairs","session_id":"791b3f05-e662-4811-b509-d772adfe51be"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.907010396Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.907025186Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"1858ce63-b815-407a-ab81-a116fd553ccf is now active with session: 791b3f05-e662-4811-b509-d772adfe51be","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.907038996Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.907050886Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248,"session_id":"791b3f05-e662-4811-b509-d772adfe51be"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.907062926Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} {"msg":"Activated sub_volume 1858ce63-b815-407a-ab81-a116fd553ccf","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:50.907073906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 150 len 8192 data: 3 4 02/30 Write block 286 len 12288 data: 3 4 3 03/30 Write block 32 len 40960 data: 2 3 3 3 4 4 4 4 3 3 04/30 Read block 11 len 28672 05/30 Read block 51 len 28672 06/30 Write block 542 len 36864 data: 3 2 2 3 3 4 3 3 3 07/30 Read block 93 len 40960 08/30 Read block 44 len 20480 09/30 Read block 579 len 32768 10/30 Read block 230 len 20480 11/30 Write block 503 len 40960 data: 2 2 3 4 4 4 4 4 3 2 12/30 Write block 312 len 36864 data: 2 2 2 3 3 2 2 2 2 13/30 Flush 14/30 Write block 419 len 24576 data: 4 4 4 4 4 4 15/30 Write block 225 len 16384 data: 2 2 3 3 16/30 Write block 521 len 40960 data: 4 3 2 2 2 2 2 2 2 2 17/30 Read block 529 len 16384 18/30 Read block 473 len 24576 19/30 Write block 522 len 24576 data: 4 3 3 3 3 3 20/30 Read block 459 len 36864 21/30 Write block 76 len 4096 data: 3 22/30 Write block 278 len 4096 data: 2 23/30 Read block 269 len 8192 24/30 Read block 312 len 32768 25/30 Write block 300 len 12288 data: 2 2 2 26/30 Write block 254 len 36864 data: 2 2 2 2 2 2 2 2 2 27/30 Read block 148 len 28672 28/30 Write block 338 len 28672 data: 3 3 3 3 2 2 2 29/30 Write block 166 len 40960 data: 2 2 2 2 2 2 2 2 2 3 30/30 Write block 495 len 36864 data: 2 2 2 2 2 2 2 2 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:50.927451363Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 10 10 10 20 20 20 T T T 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 10 10 10 20 20 20 T T T 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 13 13 13 26 26 26 T T T 12 240-259 9 9 9 17 17 17 T T T 13 260-279 12 12 12 25 25 25 T T T 14 280-299 14 14 14 27 27 27 F F F 15 300-319 14 14 14 27 27 27 T T T 16 320-339 14 14 14 27 27 27 T T T 17 340-359 13 13 13 26 26 26 T T T 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 12 12 12 25 25 25 T T T 21 420-439 13 13 13 26 26 26 T T T 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 4 4 4 6 6 6 T T T 25 500-519 14 14 14 27 27 27 T T T 26 520-539 7 7 7 11 11 11 T T T 27 540-559 14 14 14 27 27 27 F F F 28 560-579 12 12 12 25 25 25 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 14, Max flush: 27 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 15 15 15 28 28 28 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 15 15 15 28 28 28 F F F 12 240-259 15 15 15 28 28 28 F F F 13 260-279 15 15 15 28 28 28 F F F 14 280-299 14 14 14 27 27 27 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 15 15 15 28 28 28 F F F 18 360-379 9 9 9 17 17 17 F F F 19 380-399 9 9 9 17 17 17 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 15 15 15 28 28 28 F F F 25 500-519 15 15 15 28 28 28 F F F 26 520-539 15 15 15 28 28 28 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 12 12 12 25 25 25 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 15, Max flush: 28 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.205947893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.249641765Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.250227241Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.25027905Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.250301741Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.25031746Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} {"msg":"Crucible bb4786ae-24d2-4025-9dff-b4a52667e7e0 has session id: 9a815556-4b22-451c-b0c7-01204636ffe1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.250760807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Upstairs opts: Upstairs UUID: bb4786ae-24d2-4025-9dff-b4a52667e7e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.250876296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Crucible stats registered with UUID: bb4786ae-24d2-4025-9dff-b4a52667e7e0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.250911576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:05.250930116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251368582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251392232Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} {"msg":"bb4786ae-24d2-4025-9dff-b4a52667e7e0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251448932Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251476152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"0","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251505681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"0","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251520261Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"1","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251534441Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"1","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251547981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"2","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251562281Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"2","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251578741Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251603821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251629411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57596)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.25165606Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58009)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.25168312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45830)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.251909778Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.252261946Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"1","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.252441464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"0","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.25307357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"1","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.253228268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"0","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.253392767Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"2","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.253712445Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.253849574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254128162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"2","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[0]R flush_numbers[0..12]: [22, 27, 27, 28, 20, 23, 20, 27, 28, 10, 25, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254643178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[0]R generation[0..12]: [12, 14, 14, 15, 10, 12, 10, 14, 15, 6, 12, 15]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254664638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254679948Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[1]R flush_numbers[0..12]: [22, 27, 27, 28, 20, 23, 20, 27, 28, 10, 25, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254694117Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[1]R generation[0..12]: [12, 14, 14, 15, 10, 12, 10, 14, 15, 6, 12, 15]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254707757Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254721597Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[2]R flush_numbers[0..12]: [22, 27, 27, 28, 20, 23, 20, 27, 28, 10, 25, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254739807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[2]R generation[0..12]: [12, 14, 14, 15, 10, 12, 10, 14, 15, 6, 12, 15]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254753767Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254767677Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254780417Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254793477Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254806177Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254819337Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"":"downstairs","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254832316Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254844207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"0","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254857436Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"1","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254870466Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"client":"2","":"downstairs","session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254883526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254895526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"bb4786ae-24d2-4025-9dff-b4a52667e7e0 is now active with session: 9a815556-4b22-451c-b0c7-01204636ffe1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254908246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} Wait for a query_work_queue command to finish before sending IO {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254927976Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256,"session_id":"9a815556-4b22-451c-b0c7-01204636ffe1"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254941906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} {"msg":"Activated sub_volume bb4786ae-24d2-4025-9dff-b4a52667e7e0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:05.254953335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 239 len 8192 data: 2 2 02/30 Read block 358 len 20480 03/30 Write block 342 len 16384 data: 3 3 3 2 04/30 Read block 178 len 40960 05/30 Write block 279 len 28672 data: 2 2 2 2 2 3 3 06/30 Read block 332 len 8192 07/30 Read block 544 len 36864 08/30 Flush 09/30 Write block 541 len 4096 data: 3 10/30 Read block 398 len 24576 11/30 Write block 358 len 28672 data: 2 2 2 2 2 2 2 12/30 Read block 361 len 40960 13/30 Read block 217 len 36864 14/30 Write block 15 len 12288 data: 4 3 3 15/30 Read block 158 len 36864 16/30 Read block 102 len 4096 17/30 Read block 264 len 40960 18/30 Flush 19/30 Write block 252 len 4096 data: 2 20/30 Write block 567 len 16384 data: 3 4 4 4 21/30 Flush 22/30 Write block 519 len 36864 data: 4 3 5 5 4 4 4 4 4 23/30 Read block 20 len 24576 24/30 Read block 399 len 24576 25/30 Flush 26/30 Flush 27/30 Write block 288 len 28672 data: 4 3 3 3 4 4 3 28/30 Write block 561 len 8192 data: 3 4 29/30 Write block 380 len 28672 data: 4 4 4 3 3 4 4 30/30 Read block 45 len 24576 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:05.27571684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 15 15 15 28 28 28 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 31 31 31 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 16 16 16 29 29 29 T T T 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 16 16 16 30 30 30 F F F 19 380-399 9 9 9 17 17 17 T T T 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 15 15 15 28 28 28 F F F 25 500-519 16 16 16 32 32 32 F F F 26 520-539 16 16 16 32 32 32 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 16 16 16 31 31 31 T T T 29 580-599 10 10 10 19 19 19 F F F Max gen: 16, Max flush: 32 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 15 15 15 28 28 28 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 31 31 31 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 17 17 17 33 33 33 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 16 16 16 30 30 30 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 15 15 15 28 28 28 F F F 25 500-519 16 16 16 32 32 32 F F F 26 520-539 16 16 16 32 32 32 F F F 27 540-559 16 16 16 30 30 30 F F F 28 560-579 17 17 17 33 33 33 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 17, Max flush: 33 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.503058574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.546716327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.547345462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.547389052Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.547404532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.547419082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} {"msg":"Crucible f0f6ca13-7a3c-46cc-a255-9556bbafe103 has session id: d2cd781d-0493-4605-ac6f-ec7234c34647","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.547934398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Upstairs opts: Upstairs UUID: f0f6ca13-7a3c-46cc-a255-9556bbafe103, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.547975158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Crucible stats registered with UUID: f0f6ca13-7a3c-46cc-a255-9556bbafe103","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548000087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:09.548021327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548501984Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548527334Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} {"msg":"f0f6ca13-7a3c-46cc-a255-9556bbafe103 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548593963Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548611823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"0","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548640053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"0","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548653922Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"1","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548667913Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"1","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548689572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"2","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548717672Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"2","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548742962Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548770492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548799332Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"ds_connection connected from Ok(127.0.0.1:59222)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.548824571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"ds_connection connected from Ok(127.0.0.1:53319)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.54898695Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"ds_connection connected from Ok(127.0.0.1:47457)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.54903004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.549482486Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"2","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.549521896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"0","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.549896283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"1","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.550149821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"0","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.550178771Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"2","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.550510159Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"1","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.550757647Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.550856666Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[0]R flush_numbers[0..12]: [30, 27, 27, 28, 20, 23, 20, 27, 28, 10, 25, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551061184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[0]R generation[0..12]: [16, 14, 14, 15, 10, 12, 10, 14, 15, 6, 12, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551089214Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551106494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[1]R flush_numbers[0..12]: [30, 27, 27, 28, 20, 23, 20, 27, 28, 10, 25, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551125254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[1]R generation[0..12]: [16, 14, 14, 15, 10, 12, 10, 14, 15, 6, 12, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551139404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551153334Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[2]R flush_numbers[0..12]: [30, 27, 27, 28, 20, 23, 20, 27, 28, 10, 25, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551166904Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[2]R generation[0..12]: [16, 14, 14, 15, 10, 12, 10, 14, 15, 6, 12, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551180324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551194074Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551206923Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551220073Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:00:09.551232713Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551249293Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"":"downstairs","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551262383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551274113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"0","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551287213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"1","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551300263Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"client":"2","":"downstairs","session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551315053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551327422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"f0f6ca13-7a3c-46cc-a255-9556bbafe103 is now active with session: d2cd781d-0493-4605-ac6f-ec7234c34647","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551340073Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551351972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264,"session_id":"d2cd781d-0493-4605-ac6f-ec7234c34647"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551363972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Activated sub_volume f0f6ca13-7a3c-46cc-a255-9556bbafe103","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:09.551378532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 528 len 8192 data: 3 3 02/30 Read block 362 len 16384 03/30 Read block 133 len 36864 04/30 Read block 188 len 40960 05/30 Read block 582 len 4096 06/30 Flush 07/30 Write block 284 len 32768 data: 4 4 4 5 5 4 4 4 08/30 Read block 96 len 4096 09/30 Write block 123 len 36864 data: 3 4 3 2 2 2 2 3 5 10/30 Write block 483 len 20480 data: 4 4 4 4 3 11/30 Read block 195 len 20480 12/30 Read block 50 len 4096 13/30 Read block 277 len 16384 14/30 Write block 372 len 20480 data: 4 3 3 3 3 15/30 Write block 228 len 12288 data: 4 3 4 16/30 Read block 393 len 24576 17/30 Read block 131 len 32768 18/30 Read block 414 len 4096 19/30 Read block 216 len 28672 20/30 Read block 374 len 20480 21/30 Write block 519 len 36864 data: 5 4 6 6 5 5 5 5 5 22/30 Read block 221 len 32768 23/30 Read block 576 len 24576 24/30 Write block 548 len 36864 data: 4 4 4 2 3 2 2 2 2 25/30 Read block 567 len 28672 26/30 Read block 35 len 40960 27/30 Read block 295 len 20480 28/30 Read block 389 len 32768 29/30 Write block 235 len 36864 data: 2 2 2 2 3 3 2 2 3 30/30 Read block 66 len 40960 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 9 Sent Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 22 22 24 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:09.572657583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 10 10 10 20 20 20 T T T 7 140-159 14 14 14 27 27 27 F F F 8 160-179 15 15 15 28 28 28 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 16 16 16 29 29 29 T T T 12 240-259 16 16 16 31 31 31 T T T 13 260-279 16 16 16 29 29 29 F F F 14 280-299 17 17 17 33 33 33 T T T 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 16 16 16 30 30 30 T T T 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 15 15 15 28 28 28 T T T 25 500-519 16 16 16 32 32 32 T T T 26 520-539 18 18 18 34 34 34 T T T 27 540-559 16 16 16 30 30 30 T T T 28 560-579 17 17 17 33 33 33 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 18, Max flush: 34 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 30 30 30 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 19 19 19 35 35 35 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 15 15 15 28 28 28 F F F 9 180-199 6 6 6 10 10 10 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 19 19 19 35 35 35 F F F 12 240-259 19 19 19 35 35 35 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 19 19 19 35 35 35 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 19 19 19 35 35 35 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 19 19 19 35 35 35 F F F 25 500-519 19 19 19 35 35 35 F F F 26 520-539 19 19 19 35 35 35 F F F 27 540-559 19 19 19 35 35 35 F F F 28 560-579 17 17 17 33 33 33 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 19, Max flush: 35 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.830737457Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.875034716Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.875620061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.875662691Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.875677621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.87569218Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} {"msg":"Crucible a8347fda-6388-4809-b483-81ba1361c99b has session id: e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.876231616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Upstairs opts: Upstairs UUID: a8347fda-6388-4809-b483-81ba1361c99b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.876270326Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Crucible stats registered with UUID: a8347fda-6388-4809-b483-81ba1361c99b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.876294366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:13.876313686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.876801162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.876836242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} {"msg":"a8347fda-6388-4809-b483-81ba1361c99b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.876921441Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.876964871Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"0","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.877009901Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"0","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.87703879Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"1","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.8770654Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"1","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.87709001Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"2","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.87711615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"2","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.87713559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.87715039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.87716418Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:51721)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.877188309Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:45244)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.877412008Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:38626)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.877459037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.877861284Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"0","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.877973574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"2","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.878089513Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"1","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.878506419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"0","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.878534589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"2","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.878643638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"1","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879121875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879244044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[0]R flush_numbers[0..12]: [30, 27, 27, 28, 20, 23, 35, 27, 28, 10, 25, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879366133Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[0]R generation[0..12]: [16, 14, 14, 15, 10, 12, 19, 14, 15, 6, 12, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879387143Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879402503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[1]R flush_numbers[0..12]: [30, 27, 27, 28, 20, 23, 35, 27, 28, 10, 25, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879416403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[1]R generation[0..12]: [16, 14, 14, 15, 10, 12, 19, 14, 15, 6, 12, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879429773Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879444623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[2]R flush_numbers[0..12]: [30, 27, 27, 28, 20, 23, 35, 27, 28, 10, 25, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879458912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[2]R generation[0..12]: [16, 14, 14, 15, 10, 12, 19, 14, 15, 6, 12, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879472432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879486262Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879502712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879516152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:00:13.879528652Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879544062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"":"downstairs","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879556912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879568722Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"0","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879581842Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"1","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879594651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"client":"2","":"downstairs","session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879607561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879619591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"a8347fda-6388-4809-b483-81ba1361c99b is now active with session: e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879632201Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879643991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272,"session_id":"e61fe204-c5ae-4ff9-92ca-4f5f2aacc1ed"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879655921Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} {"msg":"Activated sub_volume a8347fda-6388-4809-b483-81ba1361c99b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:13.879667071Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 10 len 40960 data: 4 4 4 4 5 5 4 4 3 3 02/30 Read block 283 len 24576 03/30 Write block 539 len 36864 data: 3 3 4 4 3 3 4 4 5 04/30 Read block 554 len 32768 05/30 Read block 348 len 24576 06/30 Read block 536 len 40960 07/30 Read block 433 len 8192 08/30 Flush 09/30 Write block 491 len 40960 data: 2 2 2 2 3 3 3 3 3 3 10/30 Write block 556 len 40960 data: 3 3 3 3 3 4 5 4 4 4 11/30 Write block 221 len 12288 data: 3 3 3 12/30 Flush 13/30 Write block 228 len 36864 data: 5 4 5 3 4 3 2 3 3 14/30 Read block 9 len 36864 15/30 Flush 16/30 Write block 324 len 24576 data: 3 3 4 4 3 3 17/30 Read block 308 len 40960 18/30 Read block 434 len 12288 19/30 Write block 180 len 40960 data: 3 3 3 2 2 2 2 2 2 2 20/30 Read block 198 len 8192 21/30 Read block 589 len 12288 22/30 Read block 471 len 32768 23/30 Read block 4 len 4096 24/30 Read block 8 len 36864 25/30 Read block 571 len 24576 26/30 Read block 428 len 24576 27/30 Write block 194 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Read block 111 len 20480 29/30 Write block 177 len 12288 data: 3 3 3 30/30 Read block 476 len 40960 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 15 14 15 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:13.902285661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 19 19 19 35 35 35 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 15 15 15 28 28 28 T T T 9 180-199 6 6 6 10 10 10 T T T 10 200-219 12 12 12 25 25 25 T T T 11 220-239 20 20 20 38 38 38 F F F 12 240-259 19 19 19 35 35 35 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 19 19 19 35 35 35 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 T T T 17 340-359 16 16 16 30 30 30 F F F 18 360-379 19 19 19 35 35 35 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 20 20 20 37 37 37 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 20, Max flush: 38 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 14 14 14 27 27 27 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 10 10 10 20 20 20 F F F 5 100-119 12 12 12 23 23 23 F F F 6 120-139 19 19 19 35 35 35 F F F 7 140-159 14 14 14 27 27 27 F F F 8 160-179 21 21 21 39 39 39 F F F 9 180-199 21 21 21 39 39 39 F F F 10 200-219 21 21 21 39 39 39 F F F 11 220-239 20 20 20 38 38 38 F F F 12 240-259 19 19 19 35 35 35 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 19 19 19 35 35 35 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 21 21 21 39 39 39 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 19 19 19 35 35 35 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 13 13 13 26 26 26 F F F 23 460-479 7 7 7 11 11 11 F F F 24 480-499 20 20 20 37 37 37 F F F 25 500-519 20 20 20 37 37 37 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 20 20 20 37 37 37 F F F 28 560-579 20 20 20 37 37 37 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 21, Max flush: 39 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.131955739Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.175750971Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.176341907Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.176387626Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.176403756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.176418276Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} {"msg":"Crucible 8ab6f59e-2b9f-4743-ac9c-919811a3b9bc has session id: c6dea999-3d87-4daf-9a5d-b3abd6104d26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.176955652Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Upstairs opts: Upstairs UUID: 8ab6f59e-2b9f-4743-ac9c-919811a3b9bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.176986742Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Crucible stats registered with UUID: 8ab6f59e-2b9f-4743-ac9c-919811a3b9bc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177001251Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:18.177013961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177483188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177517838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} {"msg":"8ab6f59e-2b9f-4743-ac9c-919811a3b9bc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177598457Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177630517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"0","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177678486Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"0","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177708236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"1","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177734376Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"1","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177756656Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"2","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177780996Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"2","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177805165Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177829615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177861475Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"ds_connection connected from Ok(127.0.0.1:62171)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177890895Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"ds_connection connected from Ok(127.0.0.1:62204)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.177919145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"ds_connection connected from Ok(127.0.0.1:55684)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.178007484Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.178602899Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"2","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.178630149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"0","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.178782088Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"1","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.179308384Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"0","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.179347724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"2","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.179375724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"1","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.179977829Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180003029Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[0]R flush_numbers[0..12]: [36, 27, 27, 28, 20, 23, 35, 27, 39, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180018409Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[0]R generation[0..12]: [20, 14, 14, 15, 10, 12, 19, 14, 21, 21, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180033059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180047729Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[1]R flush_numbers[0..12]: [36, 27, 27, 28, 20, 23, 35, 27, 39, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180065179Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[1]R generation[0..12]: [20, 14, 14, 15, 10, 12, 19, 14, 21, 21, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180079119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180092818Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[2]R flush_numbers[0..12]: [36, 27, 27, 28, 20, 23, 35, 27, 39, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180106318Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[2]R generation[0..12]: [20, 14, 14, 15, 10, 12, 19, 14, 21, 21, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180119938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180133858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180146728Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180159738Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180172568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:00:18.180185788Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"":"downstairs","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180205558Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180217877Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"0","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180231047Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"1","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180259007Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"client":"2","":"downstairs","session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180275507Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180288037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"8ab6f59e-2b9f-4743-ac9c-919811a3b9bc is now active with session: c6dea999-3d87-4daf-9a5d-b3abd6104d26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180300807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180312737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280,"session_id":"c6dea999-3d87-4daf-9a5d-b3abd6104d26"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180324937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} {"msg":"Activated sub_volume 8ab6f59e-2b9f-4743-ac9c-919811a3b9bc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:18.180336097Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 446 len 16384 data: 2 2 2 3 02/30 Write block 75 len 32768 data: 4 4 3 3 3 3 2 2 03/30 Read block 122 len 40960 04/30 Read block 570 len 28672 05/30 Write block 330 len 40960 data: 3 3 3 3 3 3 3 3 4 4 06/30 Read block 485 len 4096 07/30 Read block 85 len 16384 08/30 Write block 170 len 28672 data: 3 3 3 3 3 4 3 09/30 Write block 292 len 40960 data: 5 5 4 2 2 2 2 2 3 3 10/30 Read block 555 len 40960 11/30 Flush 12/30 Write block 498 len 40960 data: 4 4 4 3 3 4 3 4 5 5 13/30 Write block 23 len 40960 data: 4 3 3 4 3 3 2 2 2 3 14/30 Write block 108 len 32768 data: 2 2 2 2 3 3 3 3 15/30 Write block 555 len 24576 data: 3 4 4 4 4 4 16/30 Write block 369 len 4096 data: 3 17/30 Write block 158 len 40960 data: 4 3 3 3 3 3 2 2 3 3 18/30 Write block 173 len 20480 data: 4 4 5 4 4 19/30 Write block 117 len 24576 data: 6 6 5 4 3 3 20/30 Read block 299 len 24576 21/30 Read block 332 len 8192 22/30 Read block 494 len 16384 23/30 Read block 30 len 20480 24/30 Read block 219 len 40960 25/30 Write block 244 len 4096 data: 3 26/30 Write block 265 len 28672 data: 3 3 3 3 3 3 3 27/30 Write block 471 len 8192 data: 2 2 28/30 Read block 339 len 36864 29/30 Read block 459 len 28672 30/30 Read block 508 len 12288 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:18.199806431Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 14 14 14 27 27 27 T T T 2 040-059 14 14 14 27 27 27 F F F 3 060-079 22 22 22 40 40 40 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 12 12 12 23 23 23 T T T 6 120-139 19 19 19 35 35 35 T T T 7 140-159 14 14 14 27 27 27 T T T 8 160-179 22 22 22 40 40 40 T T T 9 180-199 21 21 21 39 39 39 F F F 10 200-219 21 21 21 39 39 39 F F F 11 220-239 20 20 20 38 38 38 F F F 12 240-259 19 19 19 35 35 35 T T T 13 260-279 16 16 16 29 29 29 T T T 14 280-299 22 22 22 40 40 40 F F F 15 300-319 22 22 22 40 40 40 F F F 16 320-339 22 22 22 40 40 40 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 19 19 19 35 35 35 T T T 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 7 7 7 11 11 11 T T T 24 480-499 20 20 20 37 37 37 T T T 25 500-519 20 20 20 37 37 37 T T T 26 520-539 20 20 20 36 36 36 F F F 27 540-559 20 20 20 37 37 37 T T T 28 560-579 20 20 20 37 37 37 T T T 29 580-599 10 10 10 19 19 19 F F F Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 23 23 23 41 41 41 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 22 22 22 40 40 40 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 23 23 23 41 41 41 F F F 6 120-139 23 23 23 41 41 41 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 23 23 23 41 41 41 F F F 9 180-199 21 21 21 39 39 39 F F F 10 200-219 21 21 21 39 39 39 F F F 11 220-239 20 20 20 38 38 38 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 22 22 22 40 40 40 F F F 16 320-339 22 22 22 40 40 40 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 23 23 23 41 41 41 F F F 24 480-499 23 23 23 41 41 41 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 23 23 23 41 41 41 F F F 28 560-579 23 23 23 41 41 41 F F F 29 580-599 10 10 10 19 19 19 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.476653415Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:22.52015782Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.520523567Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.552484348Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.553014394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.553049023Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.553062643Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.553076793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} {"msg":"Crucible 1f8beb64-51b2-464c-9d0d-32fe04ea0cc9 has session id: 4e317337-c9e6-45ed-b58a-ee9d6aee1d3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.553613099Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Upstairs opts: Upstairs UUID: 1f8beb64-51b2-464c-9d0d-32fe04ea0cc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.553654149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Crucible stats registered with UUID: 1f8beb64-51b2-464c-9d0d-32fe04ea0cc9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.553680039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:22.553702649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554112495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554140975Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} {"msg":"1f8beb64-51b2-464c-9d0d-32fe04ea0cc9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554210995Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554229775Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"0","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554264994Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"0","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554291614Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"1","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554319224Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"1","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554344324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"2","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554394813Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"2","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554411823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554430983Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554455793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42051)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554480303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55055)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554503832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64168)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.554615802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.555235127Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"2","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.555268787Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"1","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.555395716Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"0","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.555918012Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"2","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556067131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"0","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556103671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"1","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556485928Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556659687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[0]R flush_numbers[0..12]: [36, 41, 27, 40, 40, 41, 41, 41, 41, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556722936Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[0]R generation[0..12]: [20, 23, 14, 22, 22, 23, 23, 23, 23, 21, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556742066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556757846Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[1]R flush_numbers[0..12]: [36, 41, 27, 40, 40, 41, 41, 41, 41, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556771746Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[1]R generation[0..12]: [20, 23, 14, 22, 22, 23, 23, 23, 23, 21, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556785115Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556799115Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[2]R flush_numbers[0..12]: [36, 41, 27, 40, 40, 41, 41, 41, 41, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556812655Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[2]R generation[0..12]: [20, 23, 14, 22, 22, 23, 23, 23, 23, 21, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556829515Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556843815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556856655Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556870065Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556883015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556895745Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556915825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556927714Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"0","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556940754Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"1","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556953664Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"client":"2","":"downstairs","session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556966474Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556978554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"1f8beb64-51b2-464c-9d0d-32fe04ea0cc9 is now active with session: 4e317337-c9e6-45ed-b58a-ee9d6aee1d3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.556991234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.557003044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288,"session_id":"4e317337-c9e6-45ed-b58a-ee9d6aee1d3a"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:22.557015234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} {"msg":"Activated sub_volume 1f8beb64-51b2-464c-9d0d-32fe04ea0cc9","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:00:22.557026564Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 592 len 28672 data: 2 2 2 3 3 3 3 02/30 Read block 298 len 28672 03/30 Read block 241 len 28672 04/30 Read block 468 len 24576 05/30 Write block 69 len 16384 data: 2 2 2 3 06/30 Write block 330 len 8192 data: 4 4 07/30 Read block 2 len 20480 08/30 Write block 184 len 8192 data: 3 3 09/30 Read block 87 len 28672 10/30 Write block 118 len 32768 data: 7 6 5 4 4 4 5 4 11/30 Read block 406 len 36864 12/30 Write block 165 len 36864 data: 3 4 4 3 3 4 4 4 5 13/30 Read block 534 len 12288 14/30 Write block 490 len 24576 data: 2 3 3 3 3 4 15/30 Read block 571 len 16384 16/30 Write block 263 len 24576 data: 2 3 4 4 4 4 17/30 Read block 213 len 12288 18/30 Read block 275 len 4096 19/30 Read block 550 len 16384 20/30 Read block 364 len 4096 21/30 Read block 157 len 20480 22/30 Write block 182 len 16384 data: 4 3 4 4 23/30 Read block 324 len 24576 24/30 Read block 389 len 16384 25/30 Read block 255 len 28672 26/30 Write block 26 len 8192 data: 5 4 27/30 Read block 236 len 24576 28/30 Read block 377 len 28672 29/30 Read block 133 len 12288 30/30 Write block 421 len 16384 data: 5 5 5 5 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:22.577683139Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 23 23 23 41 41 41 T T T 2 040-059 14 14 14 27 27 27 F F F 3 060-079 22 22 22 40 40 40 T T T 4 080-099 22 22 22 40 40 40 F F F 5 100-119 23 23 23 41 41 41 T T T 6 120-139 23 23 23 41 41 41 T T T 7 140-159 23 23 23 41 41 41 F F F 8 160-179 23 23 23 41 41 41 T T T 9 180-199 21 21 21 39 39 39 T T T 10 200-219 21 21 21 39 39 39 F F F 11 220-239 20 20 20 38 38 38 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 23 23 23 41 41 41 T T T 14 280-299 22 22 22 40 40 40 F F F 15 300-319 22 22 22 40 40 40 F F F 16 320-339 22 22 22 40 40 40 T T T 17 340-359 16 16 16 30 30 30 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 15 15 15 28 28 28 T T T 22 440-459 22 22 22 40 40 40 F F F 23 460-479 23 23 23 41 41 41 F F F 24 480-499 23 23 23 41 41 41 T T T 25 500-519 23 23 23 41 41 41 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 23 23 23 41 41 41 F F F 28 560-579 23 23 23 41 41 41 F F F 29 580-599 10 10 10 19 19 19 T T T Max gen: 23, Max flush: 41 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 24 24 24 42 42 42 F F F 2 040-059 14 14 14 27 27 27 F F F 3 060-079 24 24 24 42 42 42 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 24 24 24 42 42 42 F F F 6 120-139 24 24 24 42 42 42 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 42 42 42 F F F 10 200-219 21 21 21 39 39 39 F F F 11 220-239 20 20 20 38 38 38 F F F 12 240-259 23 23 23 41 41 41 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 22 22 22 40 40 40 F F F 16 320-339 24 24 24 42 42 42 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 23 23 23 41 41 41 F F F 24 480-499 24 24 24 42 42 42 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 23 23 23 41 41 41 F F F 28 560-579 23 23 23 41 41 41 F F F 29 580-599 24 24 24 42 42 42 F F F Max gen: 24, Max flush: 42 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.871112818Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.914556583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.915144568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.915196878Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.915219528Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.915237808Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} {"msg":"Crucible 6259b445-3c75-4c3d-9299-ca5c297b43c6 has session id: e32623e9-b34c-4931-8542-9f3a1dbcabce","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.915694094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Upstairs opts: Upstairs UUID: 6259b445-3c75-4c3d-9299-ca5c297b43c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.915735124Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Crucible stats registered with UUID: 6259b445-3c75-4c3d-9299-ca5c297b43c6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.915758454Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:26.915772924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916358619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916384769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} {"msg":"6259b445-3c75-4c3d-9299-ca5c297b43c6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916451428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916469858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"0","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916496638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"0","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916510548Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"1","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916532568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"1","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916562938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"2","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916591487Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"2","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916617217Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916641837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916665587Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"ds_connection connected from Ok(127.0.0.1:60545)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916691297Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"ds_connection connected from Ok(127.0.0.1:38858)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916733326Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"ds_connection connected from Ok(127.0.0.1:42349)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.916824036Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.917329412Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"2","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.917370261Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"0","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.918032507Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"0","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.918062456Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"2","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.918217025Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"1","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.918599722Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.918761111Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.9189126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"1","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[0]R flush_numbers[0..12]: [36, 42, 27, 42, 40, 42, 42, 41, 42, 42, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919425656Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[0]R generation[0..12]: [20, 24, 14, 24, 22, 24, 24, 23, 24, 24, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919454426Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919470316Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[1]R flush_numbers[0..12]: [36, 42, 27, 42, 40, 42, 42, 41, 42, 42, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919484146Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[1]R generation[0..12]: [20, 24, 14, 24, 22, 24, 24, 23, 24, 24, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919497686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919522745Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[2]R flush_numbers[0..12]: [36, 42, 27, 42, 40, 42, 42, 41, 42, 42, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919537865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[2]R generation[0..12]: [20, 24, 14, 24, 22, 24, 24, 23, 24, 24, 21, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919551595Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919565225Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Max found gen is 25","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919577875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Generation requested: 26 >= found:25","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919591005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919603675Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919622875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"":"downstairs","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919639415Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919651755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"0","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919665244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"1","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919678354Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"client":"2","":"downstairs","session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919691534Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919703504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"6259b445-3c75-4c3d-9299-ca5c297b43c6 is now active with session: e32623e9-b34c-4931-8542-9f3a1dbcabce","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919716134Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919727774Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296,"session_id":"e32623e9-b34c-4931-8542-9f3a1dbcabce"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919739884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} {"msg":"Activated sub_volume 6259b445-3c75-4c3d-9299-ca5c297b43c6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:26.919751064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 468 len 12288 data: 2 2 2 02/30 Write block 557 len 36864 data: 5 5 5 5 5 6 5 5 5 03/30 Read block 397 len 20480 04/30 Write block 281 len 4096 data: 3 05/30 Write block 72 len 20480 data: 4 4 4 5 5 06/30 Write block 300 len 24576 data: 4 4 3 2 2 2 07/30 Read block 572 len 36864 08/30 Write block 553 len 24576 data: 3 3 4 5 6 6 09/30 Read block 107 len 32768 10/30 Read block 405 len 12288 11/30 Write block 490 len 40960 data: 3 4 4 4 4 5 4 4 5 5 12/30 Read block 352 len 24576 13/30 Write block 248 len 32768 data: 3 3 3 2 3 2 3 3 14/30 Flush 15/30 Write block 454 len 16384 data: 3 3 3 3 16/30 Write block 285 len 16384 data: 5 5 6 6 17/30 Read block 212 len 16384 18/30 Write block 39 len 24576 data: 5 4 4 2 2 3 19/30 Flush 20/30 Write block 400 len 20480 data: 5 5 4 4 4 21/30 Read block 215 len 16384 22/30 Write block 211 len 28672 data: 2 2 2 3 3 4 4 23/30 Read block 497 len 4096 24/30 Read block 423 len 20480 25/30 Read block 233 len 40960 26/30 Read block 468 len 28672 27/30 Write block 247 len 36864 data: 3 4 4 4 3 4 3 4 4 28/30 Read block 54 len 32768 29/30 Read block 275 len 4096 30/30 Write block 233 len 12288 data: 4 3 4 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:26.940638697Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 26 26 26 44 44 44 F F F 2 040-059 26 26 26 44 44 44 F F F 3 060-079 26 26 26 43 43 43 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 24 24 24 42 42 42 F F F 6 120-139 24 24 24 42 42 42 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 42 42 42 F F F 10 200-219 21 21 21 39 39 39 T T T 11 220-239 20 20 20 38 38 38 T T T 12 240-259 26 26 26 43 43 43 T T T 13 260-279 24 24 24 42 42 42 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 24 24 24 42 42 42 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 15 15 15 28 28 28 T T T 21 420-439 24 24 24 42 42 42 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 26 26 26 43 43 43 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 26 26 26 43 43 43 F F F 28 560-579 26 26 26 43 43 43 F F F 29 580-599 24 24 24 42 42 42 F F F Max gen: 26, Max flush: 44 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 36 36 36 F F F 1 020-039 26 26 26 44 44 44 F F F 2 040-059 26 26 26 44 44 44 F F F 3 060-079 26 26 26 43 43 43 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 24 24 24 42 42 42 F F F 6 120-139 24 24 24 42 42 42 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 42 42 42 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 27 27 27 45 45 45 F F F 12 240-259 27 27 27 45 45 45 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 24 24 24 42 42 42 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 27 27 27 45 45 45 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 26 26 26 43 43 43 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 20 20 20 36 36 36 F F F 27 540-559 26 26 26 43 43 43 F F F 28 560-579 26 26 26 43 43 43 F F F 29 580-599 24 24 24 42 42 42 F F F Max gen: 27, Max flush: 45 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.177876259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.222670993Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.223324469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.223367218Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.223382098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.223396318Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} {"msg":"Crucible e625dfe8-3928-4f06-b782-074ab66c29ab has session id: a748e150-d686-4983-9455-d48aacafb3cf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.223942204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Upstairs opts: Upstairs UUID: e625dfe8-3928-4f06-b782-074ab66c29ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.223983944Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Crucible stats registered with UUID: e625dfe8-3928-4f06-b782-074ab66c29ab","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224007253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:31.224028493Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.22442751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.22445973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} {"msg":"e625dfe8-3928-4f06-b782-074ab66c29ab active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224518049Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224536369Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"0","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224570599Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"0","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224596599Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"1","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224623779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"1","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224652188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"2","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224680258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"2","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224705448Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224732498Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224759138Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:45765)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224785838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:52800)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224813457Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:49067)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.224879347Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.225363213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"0","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.225532642Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"1","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.225554472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"2","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226075578Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"1","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226104228Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"0","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226119577Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"2","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226690763Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226806172Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[0]R flush_numbers[0..12]: [36, 44, 44, 43, 40, 42, 42, 41, 42, 42, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226849342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[0]R generation[0..12]: [20, 26, 26, 26, 22, 24, 24, 23, 24, 24, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226866782Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226881402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[1]R flush_numbers[0..12]: [36, 44, 44, 43, 40, 42, 42, 41, 42, 42, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226895572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[1]R generation[0..12]: [20, 26, 26, 26, 22, 24, 24, 23, 24, 24, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226909071Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226923062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[2]R flush_numbers[0..12]: [36, 44, 44, 43, 40, 42, 42, 41, 42, 42, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226936461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[2]R generation[0..12]: [20, 26, 26, 26, 22, 24, 24, 23, 24, 24, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226953571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226967701Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226980651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.226993491Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.227006081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.227018901Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"":"downstairs","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.227031911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.227043621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"0","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.227056781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"1","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.22707709Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"client":"2","":"downstairs","session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.22709045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.22710243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"e625dfe8-3928-4f06-b782-074ab66c29ab is now active with session: a748e150-d686-4983-9455-d48aacafb3cf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.22711511Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.2271272Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304,"session_id":"a748e150-d686-4983-9455-d48aacafb3cf"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.22713938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} {"msg":"Activated sub_volume e625dfe8-3928-4f06-b782-074ab66c29ab","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:31.2271504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 33 len 32768 data: 4 4 4 5 5 5 6 5 02/30 Read block 234 len 16384 03/30 Read block 136 len 24576 04/30 Read block 212 len 40960 05/30 Read block 379 len 24576 06/30 Write block 254 len 40960 data: 5 5 3 3 3 3 3 3 3 3 07/30 Read block 116 len 16384 08/30 Read block 486 len 16384 09/30 Write block 6 len 16384 data: 3 4 3 4 10/30 Read block 92 len 4096 11/30 Read block 75 len 8192 12/30 Write block 539 len 8192 data: 4 4 13/30 Flush 14/30 Write block 521 len 12288 data: 7 7 6 15/30 Write block 526 len 4096 data: 6 16/30 Write block 270 len 32768 data: 4 4 3 3 3 3 2 2 17/30 Write block 589 len 16384 data: 3 3 3 3 18/30 Write block 324 len 16384 data: 4 4 5 5 19/30 Read block 186 len 40960 20/30 Read block 334 len 8192 21/30 Write block 226 len 4096 data: 3 22/30 Write block 138 len 24576 data: 3 2 2 2 2 2 23/30 Read block 419 len 32768 24/30 Write block 8 len 16384 data: 4 5 5 5 25/30 Read block 257 len 36864 26/30 Write block 51 len 28672 data: 2 2 3 3 5 5 4 27/30 Read block 210 len 24576 28/30 Read block 143 len 20480 29/30 Write block 152 len 12288 data: 4 4 5 30/30 Read block 415 len 40960 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 3 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:31.247321909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 46 46 46 T T T 1 020-039 28 28 28 46 46 46 F F F 2 040-059 28 28 28 46 46 46 T T T 3 060-079 26 26 26 43 43 43 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 24 24 24 42 42 42 F F F 6 120-139 24 24 24 42 42 42 T T T 7 140-159 23 23 23 41 41 41 T T T 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 42 42 42 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 27 27 27 45 45 45 T T T 12 240-259 28 28 28 46 46 46 F F F 13 260-279 28 28 28 46 46 46 T T T 14 280-299 26 26 26 44 44 44 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 24 24 24 42 42 42 T T T 17 340-359 16 16 16 30 30 30 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 27 27 27 45 45 45 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 26 26 26 43 43 43 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 28 28 28 46 46 46 T T T 27 540-559 28 28 28 46 46 46 F F F 28 560-579 26 26 26 43 43 43 F F F 29 580-599 24 24 24 42 42 42 T T T Max gen: 28, Max flush: 46 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 29 29 29 47 47 47 F F F 1 020-039 28 28 28 46 46 46 F F F 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 43 43 43 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 24 24 24 42 42 42 F F F 6 120-139 29 29 29 47 47 47 F F F 7 140-159 29 29 29 47 47 47 F F F 8 160-179 24 24 24 42 42 42 F F F 9 180-199 24 24 24 42 42 42 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 29 29 29 47 47 47 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 29 29 29 47 47 47 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 29 29 29 47 47 47 F F F 17 340-359 16 16 16 30 30 30 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 17 17 17 33 33 33 F F F 20 400-419 27 27 27 45 45 45 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 26 26 26 43 43 43 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 28 28 28 46 46 46 F F F 28 560-579 26 26 26 43 43 43 F F F 29 580-599 29 29 29 47 47 47 F F F Max gen: 29, Max flush: 47 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.507970981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.552355189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.552981004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.553033084Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.553054164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.553069533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} {"msg":"Crucible c3759a30-3712-43c5-87bb-aac9c3300292 has session id: b338781e-56bd-464c-857c-c32af44713dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.55356732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Upstairs opts: Upstairs UUID: c3759a30-3712-43c5-87bb-aac9c3300292, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.553613589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Crucible stats registered with UUID: c3759a30-3712-43c5-87bb-aac9c3300292","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.553639489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:45.553662719Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554113105Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554138715Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} {"msg":"c3759a30-3712-43c5-87bb-aac9c3300292 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554228405Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554246615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"0","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554274394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"0","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554288314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"1","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554302844Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"1","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554316254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"2","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554341924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"2","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554398453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554447893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554478173Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:40630)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554504483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:45588)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.554531542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33834)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.55478408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.555253107Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"2","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.555449935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"0","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.555646024Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"1","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.555980711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"0","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556012151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"2","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556287359Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"1","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556557857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556648726Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[0]R flush_numbers[0..12]: [47, 46, 47, 43, 40, 42, 47, 47, 42, 42, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556830795Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[0]R generation[0..12]: [29, 28, 29, 26, 22, 24, 29, 29, 24, 24, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556861735Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556878075Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[1]R flush_numbers[0..12]: [47, 46, 47, 43, 40, 42, 47, 47, 42, 42, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556892075Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[1]R generation[0..12]: [29, 28, 29, 26, 22, 24, 29, 29, 24, 24, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556905825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556920175Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[2]R flush_numbers[0..12]: [47, 46, 47, 43, 40, 42, 47, 47, 42, 42, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556933844Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[2]R generation[0..12]: [29, 28, 29, 26, 22, 24, 29, 29, 24, 24, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556947104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556960954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.556973634Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:00:45.556986484Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557010664Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557030634Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"":"downstairs","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557044303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557056303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"0","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557072113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"1","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557085443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"client":"2","":"downstairs","session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557098793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557111043Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"c3759a30-3712-43c5-87bb-aac9c3300292 is now active with session: b338781e-56bd-464c-857c-c32af44713dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557123743Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557135693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312,"session_id":"b338781e-56bd-464c-857c-c32af44713dd"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557147823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} {"msg":"Activated sub_volume c3759a30-3712-43c5-87bb-aac9c3300292","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:45.557158973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 16384 data: 4 4 4 2 02/30 Read block 257 len 12288 03/30 Read block 281 len 20480 04/30 Write block 515 len 4096 data: 2 05/30 Read block 286 len 32768 06/30 Read block 472 len 36864 07/30 Write block 84 len 16384 data: 3 3 3 3 08/30 Write block 349 len 40960 data: 2 2 2 2 2 2 2 2 2 3 09/30 Flush 10/30 Write block 587 len 16384 data: 2 3 4 4 11/30 Flush 12/30 Write block 165 len 20480 data: 4 5 5 4 4 13/30 Write block 349 len 12288 data: 3 3 3 14/30 Read block 310 len 32768 15/30 Write block 572 len 24576 data: 3 3 2 2 3 3 16/30 Read block 491 len 24576 17/30 Write block 166 len 20480 data: 6 6 5 5 5 18/30 Write block 314 len 32768 data: 3 4 4 3 3 3 3 2 19/30 Write block 399 len 32768 data: 3 6 6 5 5 5 5 5 20/30 Write block 31 len 24576 data: 3 4 5 5 5 6 21/30 Read block 25 len 8192 22/30 Write block 465 len 40960 data: 2 2 2 3 3 3 3 3 2 2 23/30 Write block 126 len 40960 data: 3 3 3 3 4 6 5 5 5 4 24/30 Flush 25/30 Write block 34 len 24576 data: 6 6 7 6 6 7 26/30 Read block 553 len 40960 27/30 Write block 188 len 40960 data: 3 3 2 2 2 2 3 3 3 3 28/30 Write block 94 len 28672 data: 3 3 3 3 2 2 2 29/30 Read block 128 len 20480 30/30 Read block 509 len 16384 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:45.577854878Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 29 29 29 47 47 47 F F F 1 020-039 30 30 30 50 50 50 T T T 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 43 43 43 F F F 4 080-099 30 30 30 48 48 48 T T T 5 100-119 24 24 24 42 42 42 T T T 6 120-139 30 30 30 50 50 50 F F F 7 140-159 29 29 29 47 47 47 F F F 8 160-179 30 30 30 50 50 50 F F F 9 180-199 24 24 24 42 42 42 T T T 10 200-219 27 27 27 45 45 45 F F F 11 220-239 29 29 29 47 47 47 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 29 29 29 47 47 47 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 30 30 30 50 50 50 F F F 17 340-359 30 30 30 50 50 50 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 30 30 30 50 50 50 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 30 30 30 48 48 48 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 28 28 28 46 46 46 F F F 28 560-579 30 30 30 50 50 50 F F F 29 580-599 30 30 30 49 49 49 F F F Max gen: 30, Max flush: 50 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 29 29 29 47 47 47 F F F 1 020-039 31 31 31 51 51 51 F F F 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 43 43 43 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 31 31 31 51 51 51 F F F 6 120-139 30 30 30 50 50 50 F F F 7 140-159 29 29 29 47 47 47 F F F 8 160-179 30 30 30 50 50 50 F F F 9 180-199 31 31 31 51 51 51 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 29 29 29 47 47 47 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 29 29 29 47 47 47 F F F 14 280-299 26 26 26 44 44 44 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 30 30 30 50 50 50 F F F 17 340-359 30 30 30 50 50 50 F F F 18 360-379 23 23 23 41 41 41 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 30 30 30 50 50 50 F F F 21 420-439 24 24 24 42 42 42 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 30 30 30 48 48 48 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 28 28 28 46 46 46 F F F 28 560-579 30 30 30 50 50 50 F F F 29 580-599 30 30 30 49 49 49 F F F Max gen: 31, Max flush: 51 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.804938114Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:49.847669834Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.848096111Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.879220958Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.879818204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.879852773Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.879866973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.879881843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} {"msg":"Crucible 3373b2c8-7fb4-4be4-a96c-f854d752e1c5 has session id: 3f4bd4ed-6330-4825-a1ac-7b32ca8931dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.880414939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Upstairs opts: Upstairs UUID: 3373b2c8-7fb4-4be4-a96c-f854d752e1c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.880452599Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Crucible stats registered with UUID: 3373b2c8-7fb4-4be4-a96c-f854d752e1c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.880479239Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:49.880503878Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.880911165Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.880936065Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} {"msg":"3373b2c8-7fb4-4be4-a96c-f854d752e1c5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.880999615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881019344Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"0","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881047704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"0","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881061854Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"1","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881076064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"1","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881088934Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"2","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881104504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"2","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881130104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881154574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881168853Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:33627)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881197253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:49523)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881222903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:44325)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.881446291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.882019037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"1","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.882050627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"2","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.882130676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"0","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.882705552Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"2","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.882732482Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"1","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.882800261Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"0","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883285898Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883392687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[0]R flush_numbers[0..12]: [47, 51, 47, 43, 51, 51, 50, 47, 50, 51, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883466356Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[0]R generation[0..12]: [29, 31, 29, 26, 31, 31, 30, 29, 30, 31, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883485896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883500706Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[1]R flush_numbers[0..12]: [47, 51, 47, 43, 51, 51, 50, 47, 50, 51, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883515196Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[1]R generation[0..12]: [29, 31, 29, 26, 31, 31, 30, 29, 30, 31, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883528726Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883542686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[2]R flush_numbers[0..12]: [47, 51, 47, 43, 51, 51, 50, 47, 50, 51, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883556036Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[2]R generation[0..12]: [29, 31, 29, 26, 31, 31, 30, 29, 30, 31, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883569336Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883586395Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883599655Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883612535Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883625055Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883637755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"":"downstairs","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:00:49.883650565Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883666005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"0","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883679405Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"1","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883692244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"client":"2","":"downstairs","session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883705204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883717364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"3373b2c8-7fb4-4be4-a96c-f854d752e1c5 is now active with session: 3f4bd4ed-6330-4825-a1ac-7b32ca8931dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883730014Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883741844Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320,"session_id":"3f4bd4ed-6330-4825-a1ac-7b32ca8931dc"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883753984Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} {"msg":"Activated sub_volume 3373b2c8-7fb4-4be4-a96c-f854d752e1c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:49.883764964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 248 len 16384 data: 5 5 5 4 02/30 Read block 522 len 12288 03/30 Read block 273 len 20480 04/30 Write block 278 len 32768 data: 3 3 3 4 3 3 5 6 05/30 Write block 373 len 12288 data: 4 4 4 06/30 Read block 391 len 40960 07/30 Write block 424 len 16384 data: 6 4 3 3 08/30 Flush 09/30 Write block 140 len 16384 data: 3 3 3 3 10/30 Read block 423 len 16384 11/30 Write block 118 len 32768 data: 8 7 6 5 5 5 6 5 12/30 Read block 179 len 36864 13/30 Read block 188 len 8192 14/30 Flush 15/30 Write block 21 len 24576 data: 3 3 5 4 4 6 16/30 Write block 589 len 16384 data: 5 5 4 4 17/30 Write block 428 len 40960 data: 2 3 3 4 4 3 3 4 3 4 18/30 Read block 150 len 24576 19/30 Read block 330 len 32768 20/30 Read block 472 len 4096 21/30 Flush 22/30 Write block 378 len 12288 data: 5 5 5 23/30 Read block 417 len 16384 24/30 Read block 248 len 12288 25/30 Read block 566 len 32768 26/30 Read block 228 len 8192 27/30 Write block 547 len 16384 data: 6 5 5 5 28/30 Read block 275 len 12288 29/30 Read block 252 len 12288 30/30 Write block 402 len 36864 data: 6 6 6 6 6 5 6 5 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:49.904589248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 29 29 29 47 47 47 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 43 43 43 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 32 32 53 53 53 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 30 30 30 50 50 50 F F F 9 180-199 31 31 31 51 51 51 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 29 29 29 47 47 47 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 32 32 32 52 52 52 F F F 14 280-299 32 32 32 52 52 52 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 30 30 30 50 50 50 F F F 17 340-359 30 30 30 50 50 50 F F F 18 360-379 32 32 32 52 52 52 T T T 19 380-399 30 30 30 50 50 50 T T T 20 400-419 30 30 30 50 50 50 T T T 21 420-439 32 32 32 54 54 54 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 30 30 30 48 48 48 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 28 28 28 46 46 46 T T T 28 560-579 30 30 30 50 50 50 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 32, Max flush: 54 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 29 29 29 47 47 47 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 29 29 29 47 47 47 F F F 3 060-079 26 26 26 43 43 43 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 32 32 53 53 53 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 30 30 30 50 50 50 F F F 9 180-199 31 31 31 51 51 51 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 29 29 29 47 47 47 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 32 32 32 52 52 52 F F F 14 280-299 32 32 32 52 52 52 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 30 30 30 50 50 50 F F F 17 340-359 30 30 30 50 50 50 F F F 18 360-379 33 33 33 55 55 55 F F F 19 380-399 33 33 33 55 55 55 F F F 20 400-419 33 33 33 55 55 55 F F F 21 420-439 32 32 32 54 54 54 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 26 26 26 43 43 43 F F F 25 500-519 30 30 30 48 48 48 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 33 33 33 55 55 55 F F F 28 560-579 30 30 30 50 50 50 F F F 29 580-599 32 32 32 54 54 54 F F F Max gen: 33, Max flush: 55 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.170728352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.214164046Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.214723462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.214767332Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.214782202Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.214796622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} {"msg":"Crucible 62ad2db4-1fd8-48e6-85f4-3f5fb7bec3d3 has session id: 33b1e828-1d43-4f98-b482-77c26ac29cc6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.215294198Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Upstairs opts: Upstairs UUID: 62ad2db4-1fd8-48e6-85f4-3f5fb7bec3d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.215383767Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Crucible stats registered with UUID: 62ad2db4-1fd8-48e6-85f4-3f5fb7bec3d3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.215412307Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:54.215435427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.215986242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216026462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} {"msg":"62ad2db4-1fd8-48e6-85f4-3f5fb7bec3d3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216097022Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216115341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"0","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216150281Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"0","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216177301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216206821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216234701Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"2","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216261931Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"2","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.21628478Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.21631038Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.21633357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54602)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.21636494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:54.21639543Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:54.216441439Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:54.216475939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45886)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216498559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.216520449Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.217002125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"0","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.217030245Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"2","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.21768935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"0","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.21771852Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"2","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.218378175Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:54.218403755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.217239791Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57325)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.21741089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.218538451Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219062217Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[0]R flush_numbers[0..12]: [47, 54, 47, 43, 51, 53, 53, 53, 50, 51, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219669093Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[0]R generation[0..12]: [29, 32, 29, 26, 31, 32, 32, 32, 30, 31, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219695483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219714612Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[1]R flush_numbers[0..12]: [47, 54, 47, 43, 51, 53, 53, 53, 50, 51, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219732252Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[1]R generation[0..12]: [29, 32, 29, 26, 31, 32, 32, 32, 30, 31, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219749152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219766492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[2]R flush_numbers[0..12]: [47, 54, 47, 43, 51, 53, 53, 53, 50, 51, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219783562Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[2]R generation[0..12]: [29, 32, 29, 26, 31, 32, 32, 32, 30, 31, 27, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219800352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219817392Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219834912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219851162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219866882Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219887451Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"":"downstairs","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219904241Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219919211Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"0","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219935581Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"1","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219951701Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"client":"2","":"downstairs","session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.219968291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":Wait for a query_work_queue command to finish before sending IO 1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.21999149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"62ad2db4-1fd8-48e6-85f4-3f5fb7bec3d3 is now active with session: 33b1e828-1d43-4f98-b482-77c26ac29cc6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.22000777Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.22002262Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328,"session_id":"33b1e828-1d43-4f98-b482-77c26ac29cc6"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.22003756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} {"msg":"Activated sub_volume 62ad2db4-1fd8-48e6-85f4-3f5fb7bec3d3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.2200513Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 434 len 8192 data: 4 5 02/30 Read block 141 len 32768 03/30 Write block 55 len 8192 data: 6 6 04/30 Read block 452 len 4096 05/30 Read block 472 len 36864 06/30 Flush 07/30 Write block 575 len 24576 data: 3 4 4 3 3 3 08/30 Read block 222 len 4096 09/30 Read block 182 len 36864 10/30 Write block 23 len 24576 data: 6 5 5 7 5 4 11/30 Write block 493 len 8192 data: 5 5 12/30 Read block 389 len 36864 13/30 Read block 340 len 4096 14/30 Read block 463 len 12288 15/30 Write block 43 len 24576 data: 3 4 3 3 3 2 16/30 Write block 410 len 24576 data: 4 3 3 3 2 2 17/30 Write block 361 len 32768 data: 3 3 3 3 2 2 3 3 18/30 Write block 578 len 12288 data: 4 4 4 19/30 Read block 72 len 36864 20/30 Write block 227 len 8192 data: 4 6 21/30 Read block 269 len 40960 22/30 Write block 1 len 36864 data: 4 4 4 4 4 4 5 5 6 23/30 Write block 71 len 12288 data: 3 5 5 24/30 Flush 25/30 Write block 323 len 40960 data: 2 5 5 6 6 4 4 5 5 4 26/30 Read block 59 len 16384 27/30 Read block 39 len 12288 28/30 Write block 266 len 32768 data: 5 5 5 4 5 5 4 4 29/30 Write block 564 len 40960 data: 6 6 3 4 5 5 5 3 4 4 30/30 Read block 596 len 16384 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:04.240434837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 34 34 34 57 57 57 F F F 3 060-079 34 34 34 57 57 57 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 32 32 53 53 53 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 30 30 30 50 50 50 F F F 9 180-199 31 31 31 51 51 51 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 32 32 32 52 52 52 T T T 14 280-299 32 32 32 52 52 52 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 30 30 30 50 50 50 T T T 17 340-359 30 30 30 50 50 50 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 33 33 33 55 55 55 F F F 20 400-419 34 34 34 57 57 57 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 30 30 30 48 48 48 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 33 33 33 55 55 55 F F F 28 560-579 34 34 34 57 57 57 T T T 29 580-599 34 34 34 57 57 57 F F F Max gen: 34, Max flush: 57 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 34 34 34 57 57 57 F F F 3 060-079 34 34 34 57 57 57 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 32 32 32 53 53 53 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 F F F 8 160-179 30 30 30 50 50 50 F F F 9 180-199 31 31 31 51 51 51 F F F 10 200-219 27 27 27 45 45 45 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 35 35 35 58 58 58 F F F 14 280-299 32 32 32 52 52 52 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 30 30 30 50 50 50 F F F 18 360-379 34 34 34 57 57 57 F F F 19 380-399 33 33 33 55 55 55 F F F 20 400-419 34 34 34 57 57 57 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 34 34 34 57 57 57 F F F 25 500-519 30 30 30 48 48 48 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 33 33 33 55 55 55 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 57 57 57 F F F Max gen: 35, Max flush: 58 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.46668041Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.51079264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.511385596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.511428905Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.511443865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.511458305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} {"msg":"Crucible 80d51b97-3926-4090-bea9-75be6b5073bb has session id: 30995cac-0bd1-4eee-a0aa-bf7bd034f5a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.511958141Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Upstairs opts: Upstairs UUID: 80d51b97-3926-4090-bea9-75be6b5073bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.511987461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Crucible stats registered with UUID: 80d51b97-3926-4090-bea9-75be6b5073bb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512008991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:08.512027781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512508687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512581737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} {"msg":"80d51b97-3926-4090-bea9-75be6b5073bb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512620786Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512646066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"0","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512691226Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"0","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512721586Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"1","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512750836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"1","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512777795Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"2","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512806105Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"2","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512835675Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512861855Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512886744Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63034)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512913734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62871)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512941924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47333)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.512966494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.513656379Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"0","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.513697318Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"2","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.514087846Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"1","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.514248914Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"2","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.514278934Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"0","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.514612222Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"1","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.51483053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.51485554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[0]R flush_numbers[0..12]: [57, 57, 57, 57, 51, 53, 53, 53, 50, 51, 45, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515143038Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[0]R generation[0..12]: [34, 34, 34, 34, 31, 32, 32, 32, 30, 31, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515171257Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515186737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[1]R flush_numbers[0..12]: [57, 57, 57, 57, 51, 53, 53, 53, 50, 51, 45, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515201117Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[1]R generation[0..12]: [34, 34, 34, 34, 31, 32, 32, 32, 30, 31, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515215047Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515229117Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[2]R flush_numbers[0..12]: [57, 57, 57, 57, 51, 53, 53, 53, 50, 51, 45, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515242547Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[2]R generation[0..12]: [34, 34, 34, 34, 31, 32, 32, 32, 30, 31, 27, 34]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515259327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515273326Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515286226Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2025-03-18T05:01:08.515299276Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515320126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515332966Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"":"downstairs","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515345936Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515357796Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"0","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515371066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"1","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515384086Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"client":"2","":"downstairs","session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515397196Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515409345Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"80d51b97-3926-4090-bea9-75be6b5073bb is now active with session: 30995cac-0bd1-4eee-a0aa-bf7bd034f5a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515422136Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515433965Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336,"session_id":"30995cac-0bd1-4eee-a0aa-bf7bd034f5a8"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515446135Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} {"msg":"Activated sub_volume 80d51b97-3926-4090-bea9-75be6b5073bb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.515457645Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 36864 data: 5 5 4 5 6 4 5 4 4 02/30 Flush 03/30 Write block 49 len 24576 data: 2 2 3 3 4 4 04/30 Write block 403 len 36864 data: 7 7 7 7 6 7 6 5 4 05/30 Read block 27 len 40960 06/30 Read block 127 len 16384 07/30 Write block 292 len 4096 data: 6 08/30 Write block 64 len 28672 data: 3 2 2 2 2 3 3 09/30 Write block 271 len 12288 data: 6 5 5 10/30 Write block 370 len 20480 data: 4 4 5 5 5 11/30 Write block 103 len 20480 data: 2 2 2 2 2 12/30 Write block 546 len 8192 data: 5 7 13/30 Read block 104 len 28672 14/30 Flush 15/30 Write block 491 len 8192 data: 5 5 16/30 Read block 213 len 4096 17/30 Write block 175 len 36864 data: 6 5 5 4 4 4 4 5 4 18/30 Read block 560 len 32768 19/30 Flush 20/30 Write block 510 len 16384 data: 5 4 3 2 21/30 Write block 189 len 12288 data: 4 3 3 22/30 Flush 23/30 Write block 156 len 20480 data: 3 3 5 4 4 24/30 Read block 164 len 4096 25/30 Write block 58 len 4096 data: 4 26/30 Write block 197 len 36864 data: 4 3 3 3 3 2 2 2 3 27/30 Write block 398 len 28672 data: 3 4 7 7 7 8 8 28/30 Read block 427 len 28672 29/30 Read block 50 len 12288 30/30 Read block 144 len 36864 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 7 7 8 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:08.5348109Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 36 36 36 60 60 60 T T T 3 060-079 36 36 36 60 60 60 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 36 36 36 60 60 60 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 32 32 32 53 53 53 T T T 8 160-179 36 36 36 61 61 61 T T T 9 180-199 36 36 36 62 62 62 T T T 10 200-219 27 27 27 45 45 45 T T T 11 220-239 34 34 34 57 57 57 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 36 36 36 60 60 60 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 30 30 30 50 50 50 F F F 18 360-379 36 36 36 60 60 60 F F F 19 380-399 33 33 33 55 55 55 T T T 20 400-419 36 36 36 60 60 60 T T T 21 420-439 36 36 36 59 59 59 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 36 36 36 62 62 62 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 36 36 36 60 60 60 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 57 57 57 F F F Max gen: 36, Max flush: 62 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 37 37 37 63 63 63 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 36 36 36 60 60 60 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 37 37 37 63 63 63 F F F 8 160-179 37 37 37 63 63 63 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 37 37 37 63 63 63 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 36 36 36 60 60 60 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 30 30 30 50 50 50 F F F 18 360-379 36 36 36 60 60 60 F F F 19 380-399 37 37 37 63 63 63 F F F 20 400-419 37 37 37 63 63 63 F F F 21 420-439 36 36 36 59 59 59 F F F 22 440-459 26 26 26 44 44 44 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 36 36 36 62 62 62 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 36 36 36 60 60 60 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 57 57 57 F F F Max gen: 37, Max flush: 63 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.781548279Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:12.824437988Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.824809335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.855908173Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.856463408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.856505288Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.856523718Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.856538598Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} {"msg":"Crucible 6cd69187-4a76-4440-9a74-339287953b34 has session id: fcc60ef5-6e1f-474b-9959-a3a6ede379b8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857191373Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Upstairs opts: Upstairs UUID: 6cd69187-4a76-4440-9a74-339287953b34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857240492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Crucible stats registered with UUID: 6cd69187-4a76-4440-9a74-339287953b34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857261452Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:12.857278022Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857699669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857726229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} {"msg":"6cd69187-4a76-4440-9a74-339287953b34 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857790668Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857808268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"0","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857836278Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"0","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857850348Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"1","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857864868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"1","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857878008Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"2","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857892848Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"2","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857910547Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857926758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857950587Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52751)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.857997347Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37831)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.858225515Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54022)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.858261455Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.858776371Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"2","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.858819291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"1","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.858849611Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"0","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.859458406Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"1","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.859499776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"0","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.859531186Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"2","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860057082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860174651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[0]R flush_numbers[0..12]: [57, 57, 63, 60, 51, 60, 53, 63, 63, 63, 63, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.86020569Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[0]R generation[0..12]: [34, 34, 37, 36, 31, 36, 32, 37, 37, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.86022849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.86024346Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[1]R flush_numbers[0..12]: [57, 57, 63, 60, 51, 60, 53, 63, 63, 63, 63, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.86025712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[1]R generation[0..12]: [34, 34, 37, 36, 31, 36, 32, 37, 37, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.86027072Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.86028459Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[2]R flush_numbers[0..12]: [57, 57, 63, 60, 51, 60, 53, 63, 63, 63, 63, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.86029902Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[2]R generation[0..12]: [34, 34, 37, 36, 31, 36, 32, 37, 37, 37, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860313229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860327209Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860340909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860353989Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860366759Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860388699Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"":"downstairs","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860414349Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860439289Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"0","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860476208Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"1","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860505008Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"client":"2","":"downstairs","session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860531708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860557548Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"6cd69187-4a76-4440-9a74-339287953b34 is now active with session: fcc60ef5-6e1f-474b-9959-a3a6ede379b8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860583527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860607827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344,"session_id":"fcc60ef5-6e1f-474b-9959-a3a6ede379b8"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860633007Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Activated sub_volume 6cd69187-4a76-4440-9a74-339287953b34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:12.860659627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 16384 data: 6 6 5 6 02/30 Read block 269 len 8192 03/30 Read block 12 len 32768 04/30 Read block 34 len 32768 05/30 Read block 117 len 20480 06/30 Read block 477 len 8192 07/30 Write block 500 len 4096 data: 5 08/30 Flush 09/30 Flush 10/30 Write block 3 len 4096 data: 5 11/30 Write block 565 len 8192 data: 7 4 12/30 Read block 362 len 12288 13/30 Flush 14/30 Write block 9 len 28672 data: 7 6 6 5 5 6 6 15/30 Write block 235 len 28672 data: 5 4 3 3 4 4 3 16/30 Flush 17/30 Write block 351 len 8192 data: 4 3 18/30 Flush 19/30 Write block 555 len 36864 data: 5 6 7 7 6 6 6 7 6 20/30 Read block 274 len 36864 21/30 Write block 482 len 24576 data: 4 5 5 5 5 4 22/30 Read block 282 len 4096 23/30 Read block 289 len 4096 24/30 Write block 437 len 16384 data: 6 5 5 4 25/30 Flush 26/30 Write block 370 len 8192 data: 5 5 27/30 Write block 289 len 20480 data: 5 5 5 7 6 28/30 Write block 55 len 28672 data: 7 7 5 5 4 4 4 29/30 Read block 542 len 12288 30/30 Read block 515 len 32768 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:12.879344827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 37 37 37 63 63 63 T T T 3 060-079 36 36 36 60 60 60 T T T 4 080-099 31 31 31 51 51 51 F F F 5 100-119 36 36 36 60 60 60 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 37 37 37 63 63 63 F F F 8 160-179 37 37 37 63 63 63 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 37 37 37 63 63 63 F F F 11 220-239 38 38 38 67 67 67 F F F 12 240-259 38 38 38 67 67 67 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 36 36 36 60 60 60 T T T 15 300-319 30 30 30 50 50 50 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 38 38 38 68 68 68 F F F 18 360-379 36 36 36 60 60 60 T T T 19 380-399 37 37 37 63 63 63 F F F 20 400-419 37 37 37 63 63 63 F F F 21 420-439 38 38 38 69 69 69 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 38 38 38 64 64 64 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 38 38 38 69 69 69 F F F 28 560-579 38 38 38 69 69 69 F F F 29 580-599 34 34 34 57 57 57 F F F Max gen: 38, Max flush: 69 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 39 39 39 70 70 70 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 36 36 36 60 60 60 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 37 37 37 63 63 63 F F F 8 160-179 37 37 37 63 63 63 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 37 37 37 63 63 63 F F F 11 220-239 38 38 38 67 67 67 F F F 12 240-259 38 38 38 67 67 67 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 39 39 39 70 70 70 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 38 38 38 68 68 68 F F F 18 360-379 39 39 39 70 70 70 F F F 19 380-399 37 37 37 63 63 63 F F F 20 400-419 37 37 37 63 63 63 F F F 21 420-439 38 38 38 69 69 69 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 38 38 38 64 64 64 F F F 26 520-539 29 29 29 47 47 47 F F F 27 540-559 38 38 38 69 69 69 F F F 28 560-579 38 38 38 69 69 69 F F F 29 580-599 34 34 34 57 57 57 F F F Max gen: 39, Max flush: 70 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.139899783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.183445357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.184056163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.184096332Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.184111322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.184125502Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} {"msg":"Crucible f7045b07-586e-47b5-b119-a4c2a2327698 has session id: 4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.184746338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Upstairs opts: Upstairs UUID: f7045b07-586e-47b5-b119-a4c2a2327698, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.184777087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Crucible stats registered with UUID: f7045b07-586e-47b5-b119-a4c2a2327698","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.184792667Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:17.184805967Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185317343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185341623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} {"msg":"f7045b07-586e-47b5-b119-a4c2a2327698 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185414853Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185433102Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"0","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185460462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"0","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185478612Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185497162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185521302Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"2","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185548072Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"2","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185563051Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185585411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185608361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55375)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.18575634Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:17.185879569Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:17.185923609Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43883)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.185940938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:17.185957138Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.186087288Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.186523754Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"0","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.186555644Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"2","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.187174259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"0","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.187203709Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"2","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.187851254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.187877714Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.186904259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:42642)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.186986849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.18815918Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.188874674Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[0]R flush_numbers[0..12]: [67, 57, 70, 70, 51, 60, 53, 63, 63, 63, 63, 67]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189580919Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[0]R generation[0..12]: [38, 34, 39, 39, 31, 36, 32, 37, 37, 37, 37, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189609349Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189625289Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[1]R flush_numbers[0..12]: [67, 57, 70, 70, 51, 60, 53, 63, 63, 63, 63, 67]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189639389Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[1]R generation[0..12]: [38, 34, 39, 39, 31, 36, 32, 37, 37, 37, 37, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189653009Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189673059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[2]R flush_numbers[0..12]: [67, 57, 70, 70, 51, 60, 53, 63, 63, 63, 63, 67]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189687128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[2]R generation[0..12]: [38, 34, 39, 39, 31, 36, 32, 37, 37, 37, 37, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189701248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189715338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189728018Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189740788Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189753288Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189765938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"":"downstairs","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189778948Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189790938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"0","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189804258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"1","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189817107Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"client":"2","":"downstairs","session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"All required reconciliation work is completed","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2025-03-18T05:01:27.189837187Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189849417Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"f7045b07-586e-47b5-b119-a4c2a2327698 is now active with session: 4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189862087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189876537Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352,"session_id":"4f6f8a7f-22b5-4d69-a665-1cdd5f4029f2"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189888927Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} {"msg":"Activated sub_volume f7045b07-586e-47b5-b119-a4c2a2327698","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:27.189899997Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 116 len 8192 data: 5 7 02/30 Read block 445 len 24576 03/30 Write block 221 len 4096 data: 4 04/30 Write block 131 len 40960 data: 7 6 6 6 5 4 4 4 3 4 05/30 Flush 06/30 Write block 430 len 40960 data: 4 7 7 6 7 7 5 7 6 6 07/30 Write block 592 len 16384 data: 5 3 3 4 08/30 Read block 10 len 8192 09/30 Read block 125 len 16384 10/30 Read block 428 len 32768 11/30 Write block 389 len 16384 data: 3 2 2 2 12/30 Write block 299 len 36864 data: 3 5 5 4 3 3 3 2 2 13/30 Read block 245 len 12288 14/30 Read block 71 len 4096 15/30 Write block 140 len 4096 data: 5 16/30 Read block 526 len 32768 17/30 Read block 51 len 12288 18/30 Read block 85 len 12288 19/30 Read block 273 len 24576 20/30 Read block 412 len 40960 21/30 Read block 294 len 24576 22/30 Read block 284 len 20480 23/30 Read block 304 len 24576 24/30 Write block 528 len 36864 data: 4 4 3 2 2 2 2 2 2 25/30 Read block 270 len 4096 26/30 Write block 568 len 40960 data: 6 6 6 4 5 5 3 4 5 5 27/30 Read block 10 len 32768 28/30 Write block 502 len 40960 data: 4 5 4 5 6 6 5 5 6 5 29/30 Read block 4 len 32768 30/30 Read block 166 len 28672 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 10 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 25 25 24 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:27.211546545Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 39 39 39 70 70 70 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 40 40 40 71 71 71 T T T 8 160-179 37 37 37 63 63 63 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 37 37 37 63 63 63 F F F 11 220-239 40 40 40 71 71 71 F F F 12 240-259 38 38 38 67 67 67 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 39 39 39 70 70 70 T T T 15 300-319 30 30 30 50 50 50 T T T 16 320-339 35 35 35 58 58 58 F F F 17 340-359 38 38 38 68 68 68 F F F 18 360-379 39 39 39 70 70 70 F F F 19 380-399 37 37 37 63 63 63 T T T 20 400-419 37 37 37 63 63 63 F F F 21 420-439 38 38 38 69 69 69 T T T 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 38 38 38 64 64 64 T T T 26 520-539 29 29 29 47 47 47 T T T 27 540-559 38 38 38 69 69 69 F F F 28 560-579 38 38 38 69 69 69 T T T 29 580-599 34 34 34 57 57 57 T T T Max gen: 40, Max flush: 71 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 34 34 34 57 57 57 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 39 39 39 70 70 70 F F F 4 080-099 31 31 31 51 51 51 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 41 41 41 72 72 72 F F F 8 160-179 37 37 37 63 63 63 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 37 37 37 63 63 63 F F F 11 220-239 40 40 40 71 71 71 F F F 12 240-259 38 38 38 67 67 67 F F F 13 260-279 36 36 36 60 60 60 F F F 14 280-299 41 41 41 72 72 72 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 38 38 38 68 68 68 F F F 18 360-379 39 39 39 70 70 70 F F F 19 380-399 41 41 41 72 72 72 F F F 20 400-419 37 37 37 63 63 63 F F F 21 420-439 41 41 41 72 72 72 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 38 38 38 69 69 69 F F F 25 500-519 41 41 41 72 72 72 F F F 26 520-539 41 41 41 72 72 72 F F F 27 540-559 38 38 38 69 69 69 F F F 28 560-579 41 41 41 72 72 72 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 41, Max flush: 72 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.47478552Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.519481206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.520096601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.520135721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.520149651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.520163711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} {"msg":"Crucible 1ff784e1-92cf-4814-a82f-9d0d2d01866e has session id: afced2b4-653a-4101-8928-8d94fa630ab2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.520745846Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Upstairs opts: Upstairs UUID: 1ff784e1-92cf-4814-a82f-9d0d2d01866e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.520785686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Crucible stats registered with UUID: 1ff784e1-92cf-4814-a82f-9d0d2d01866e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.520809256Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:31.520823766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521205553Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521228963Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} {"msg":"1ff784e1-92cf-4814-a82f-9d0d2d01866e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521294862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521313192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"0","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521339852Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"0","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521356982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"1","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521371462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"1","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521384642Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"2","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521398502Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"2","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521411581Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521433911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521459021Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"ds_connection connected from Ok(127.0.0.1:64439)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.521483431Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"ds_connection connected from Ok(127.0.0.1:64278)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.52152591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62167)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.52155454Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.522291255Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"0","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.522319835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"2","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.522335084Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"1","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.5229356Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"0","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.52296453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"1","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.522990259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"2","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523505476Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523532855Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[0]R flush_numbers[0..12]: [67, 57, 70, 70, 51, 71, 71, 72, 63, 63, 63, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523549285Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[0]R generation[0..12]: [38, 34, 39, 39, 31, 40, 40, 41, 37, 37, 37, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523564245Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523578375Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[1]R flush_numbers[0..12]: [67, 57, 70, 70, 51, 71, 71, 72, 63, 63, 63, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523592305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[1]R generation[0..12]: [38, 34, 39, 39, 31, 40, 40, 41, 37, 37, 37, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523606045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523619755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[2]R flush_numbers[0..12]: [67, 57, 70, 70, 51, 71, 71, 72, 63, 63, 63, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523633335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[2]R generation[0..12]: [38, 34, 39, 39, 31, 40, 40, 41, 37, 37, 37, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523646845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523660734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523673434Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523686414Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523702324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523715204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"":"downstairs","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523728234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523740194Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"0","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523753264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"1","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523766184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"client":"2","":"downstairs","session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:31.523779184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523796224Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"1ff784e1-92cf-4814-a82f-9d0d2d01866e is now active with session: afced2b4-653a-4101-8928-8d94fa630ab2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523811393Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523829393Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360,"session_id":"afced2b4-653a-4101-8928-8d94fa630ab2"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523842243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} {"msg":"Activated sub_volume 1ff784e1-92cf-4814-a82f-9d0d2d01866e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:31.523853723Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 40960 data: 3 3 3 3 3 3 4 4 5 5 02/30 Write block 497 len 32768 data: 5 6 6 6 4 5 6 5 03/30 Read block 386 len 12288 04/30 Read block 232 len 12288 05/30 Read block 570 len 32768 06/30 Write block 36 len 12288 data: 8 7 7 07/30 Read block 175 len 28672 08/30 Write block 366 len 20480 data: 3 4 4 4 6 09/30 Read block 466 len 24576 10/30 Read block 239 len 40960 11/30 Write block 359 len 4096 data: 3 12/30 Write block 89 len 20480 data: 4 5 3 3 3 13/30 Read block 552 len 36864 14/30 Read block 428 len 32768 15/30 Read block 535 len 8192 16/30 Read block 49 len 8192 17/30 Write block 71 len 36864 data: 4 6 6 5 6 6 4 4 4 18/30 Read block 175 len 32768 19/30 Write block 74 len 32768 data: 6 7 7 5 5 5 4 3 20/30 Read block 263 len 28672 21/30 Read block 105 len 4096 22/30 Read block 270 len 20480 23/30 Read block 497 len 4096 24/30 Read block 173 len 8192 25/30 Read block 170 len 16384 26/30 Write block 272 len 16384 data: 6 6 4 4 27/30 Read block 411 len 16384 28/30 Read block 86 len 32768 29/30 Read block 320 len 12288 30/30 Read block 106 len 36864 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:31.545051154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 34 34 34 57 57 57 T T T 2 040-059 39 39 39 70 70 70 F F F 3 060-079 39 39 39 70 70 70 T T T 4 080-099 31 31 31 51 51 51 T T T 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 41 41 41 72 72 72 F F F 8 160-179 37 37 37 63 63 63 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 37 37 37 63 63 63 T T T 11 220-239 40 40 40 71 71 71 F F F 12 240-259 38 38 38 67 67 67 F F F 13 260-279 36 36 36 60 60 60 T T T 14 280-299 41 41 41 72 72 72 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 38 38 38 68 68 68 T T T 18 360-379 39 39 39 70 70 70 T T T 19 380-399 41 41 41 72 72 72 F F F 20 400-419 37 37 37 63 63 63 F F F 21 420-439 41 41 41 72 72 72 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 38 38 38 69 69 69 T T T 25 500-519 41 41 41 72 72 72 T T T 26 520-539 41 41 41 72 72 72 F F F 27 540-559 38 38 38 69 69 69 F F F 28 560-579 41 41 41 72 72 72 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 41, Max flush: 72 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 42 42 42 73 73 73 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 42 42 42 73 73 73 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 41 41 41 72 72 72 F F F 8 160-179 37 37 37 63 63 63 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 42 42 42 73 73 73 F F F 11 220-239 40 40 40 71 71 71 F F F 12 240-259 38 38 38 67 67 67 F F F 13 260-279 42 42 42 73 73 73 F F F 14 280-299 41 41 41 72 72 72 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 35 35 35 58 58 58 F F F 17 340-359 42 42 42 73 73 73 F F F 18 360-379 42 42 42 73 73 73 F F F 19 380-399 41 41 41 72 72 72 F F F 20 400-419 37 37 37 63 63 63 F F F 21 420-439 41 41 41 72 72 72 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 42 42 42 73 73 73 F F F 25 500-519 42 42 42 73 73 73 F F F 26 520-539 41 41 41 72 72 72 F F F 27 540-559 38 38 38 69 69 69 F F F 28 560-579 41 41 41 72 72 72 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 42, Max flush: 73 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.802163076Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.846164587Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.846786352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.846830532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.846846541Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.846862012Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} {"msg":"Crucible 1fc34dce-4e0a-4ad0-a78c-7ee073db2c5f has session id: 7675e739-d40f-44cd-9036-cf37d921d73e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.847432957Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Upstairs opts: Upstairs UUID: 1fc34dce-4e0a-4ad0-a78c-7ee073db2c5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.847474427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Crucible stats registered with UUID: 1fc34dce-4e0a-4ad0-a78c-7ee073db2c5f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.847500477Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:35.847523027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.847968253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.847996533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} {"msg":"1fc34dce-4e0a-4ad0-a78c-7ee073db2c5f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848066263Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848102882Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"0","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848147102Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"0","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848173252Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"1","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848200491Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"1","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848225061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"2","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848251561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"2","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848276271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848303101Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.84833485Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51071)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.8483733Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37209)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848532339Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54798)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.848571489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.849062495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"2","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.849092455Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"0","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.84973875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"2","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.849877809Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"1","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.849915758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"0","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.850385815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.850418265Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.850434305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"1","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[0]R flush_numbers[0..12]: [67, 73, 70, 73, 73, 71, 71, 72, 63, 63, 73, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.850994781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[0]R generation[0..12]: [38, 42, 39, 42, 42, 40, 40, 41, 37, 37, 42, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85101769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85103727Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[1]R flush_numbers[0..12]: [67, 73, 70, 73, 73, 71, 71, 72, 63, 63, 73, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85105227Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[1]R generation[0..12]: [38, 42, 39, 42, 42, 40, 40, 41, 37, 37, 42, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85106582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85107972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[2]R flush_numbers[0..12]: [67, 73, 70, 73, 73, 71, 71, 72, 63, 63, 73, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85109319Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[2]R generation[0..12]: [38, 42, 39, 42, 42, 40, 40, 41, 37, 37, 42, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85110659Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.85112028Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851133229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851147419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851160259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:35.851172969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"":"downstairs","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851192579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851204419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"0","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851217789Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"1","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851230849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"client":"2","":"downstairs","session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851247339Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851260759Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"1fc34dce-4e0a-4ad0-a78c-7ee073db2c5f is now active with session: 7675e739-d40f-44cd-9036-cf37d921d73e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851273568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851285528Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368,"session_id":"7675e739-d40f-44cd-9036-cf37d921d73e"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851297498Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} {"msg":"Activated sub_volume 1fc34dce-4e0a-4ad0-a78c-7ee073db2c5f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.851308408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 12288 data: 8 8 7 02/30 Write block 86 len 12288 data: 4 4 3 03/30 Write block 158 len 36864 data: 6 5 5 4 4 4 3 5 7 04/30 Write block 557 len 20480 data: 8 8 7 7 7 05/30 Flush 06/30 Flush 07/30 Write block 31 len 24576 data: 4 5 6 7 7 9 08/30 Read block 43 len 4096 09/30 Write block 374 len 32768 data: 6 5 4 3 6 6 6 5 10/30 Read block 335 len 28672 11/30 Write block 403 len 24576 data: 9 9 8 8 7 8 12/30 Read block 283 len 20480 13/30 Read block 364 len 28672 14/30 Read block 172 len 36864 15/30 Write block 332 len 36864 data: 5 4 4 4 4 4 5 5 4 16/30 Read block 456 len 32768 17/30 Read block 223 len 12288 18/30 Write block 424 len 32768 data: 7 5 4 4 3 4 5 9 19/30 Read block 439 len 12288 20/30 Read block 166 len 16384 21/30 Write block 268 len 20480 data: 6 5 6 7 7 22/30 Write block 353 len 20480 data: 3 3 3 3 3 23/30 Read block 432 len 36864 24/30 Write block 224 len 4096 data: 2 25/30 Write block 236 len 32768 data: 5 4 4 5 5 4 3 4 26/30 Flush 27/30 Write block 221 len 20480 data: 5 4 4 3 3 28/30 Read block 333 len 28672 29/30 Write block 87 len 40960 data: 5 4 5 6 4 4 4 4 4 4 30/30 Write block 210 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:35.872367941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 44 44 44 74 74 74 T T T 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 44 44 44 74 74 74 F F F 8 160-179 44 44 44 74 74 74 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 42 42 42 73 73 73 T T T 11 220-239 44 44 44 76 76 76 T T T 12 240-259 44 44 44 76 76 76 F F F 13 260-279 44 44 44 76 76 76 F F F 14 280-299 41 41 41 72 72 72 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 44 44 44 76 76 76 F F F 17 340-359 44 44 44 76 76 76 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 44 44 44 76 76 76 F F F 20 400-419 44 44 44 76 76 76 F F F 21 420-439 44 44 44 76 76 76 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 42 42 42 73 73 73 F F F 25 500-519 42 42 42 73 73 73 F F F 26 520-539 41 41 41 72 72 72 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 44, Max flush: 76 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 44 44 44 74 74 74 F F F 8 160-179 44 44 44 74 74 74 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 45 45 45 77 77 77 F F F 12 240-259 44 44 44 76 76 76 F F F 13 260-279 44 44 44 76 76 76 F F F 14 280-299 41 41 41 72 72 72 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 44 44 44 76 76 76 F F F 17 340-359 44 44 44 76 76 76 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 44 44 44 76 76 76 F F F 20 400-419 44 44 44 76 76 76 F F F 21 420-439 44 44 44 76 76 76 F F F 22 440-459 38 38 38 69 69 69 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 42 42 42 73 73 73 F F F 25 500-519 42 42 42 73 73 73 F F F 26 520-539 41 41 41 72 72 72 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 45, Max flush: 77 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.098737483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.142187398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.142754644Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.142795264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.142810023Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.142824213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} {"msg":"Crucible 57a07404-f8db-4570-91cf-1929a5d010fb has session id: e1d4f42e-2a31-4f5a-90fb-733be722b134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.143477868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Upstairs opts: Upstairs UUID: 57a07404-f8db-4570-91cf-1929a5d010fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.143507508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Crucible stats registered with UUID: 57a07404-f8db-4570-91cf-1929a5d010fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.143532188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:40.143556928Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.143997684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144027674Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} {"msg":"57a07404-f8db-4570-91cf-1929a5d010fb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144107264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144125194Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"0","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144152673Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"0","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144166393Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"1","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144180603Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"1","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144207203Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"2","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144250833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"2","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144277992Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144306342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144331582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"ds_connection connected from Ok(127.0.0.1:37345)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144356772Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"ds_connection connected from Ok(127.0.0.1:57957)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144383992Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"ds_connection connected from Ok(127.0.0.1:63113)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.144410321Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.145089226Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"2","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.145131196Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"0","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.145819961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"2","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14585205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"0","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14594028Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"1","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.146486856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.146517576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"1","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.146533956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[0]R flush_numbers[0..12]: [67, 76, 70, 73, 77, 71, 71, 74, 74, 63, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147045442Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[0]R generation[0..12]: [38, 44, 39, 42, 45, 40, 40, 44, 44, 37, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147064372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147079061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[1]R flush_numbers[0..12]: [67, 76, 70, 73, 77, 71, 71, 74, 74, 63, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147093111Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[1]R generation[0..12]: [38, 44, 39, 42, 45, 40, 40, 44, 44, 37, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147107661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147121641Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[2]R flush_numbers[0..12]: [67, 76, 70, 73, 77, 71, 71, 74, 74, 63, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147135141Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[2]R generation[0..12]: [38, 44, 39, 42, 45, 40, 40, 44, 44, 37, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147148521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147162351Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147175431Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14718846Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147201331Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14722367Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"":"downstairs","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14723673Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14724858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"0","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.1472617Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"1","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14727494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"client":"2","":"downstairs","session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14728802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14730016Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"57a07404-f8db-4570-91cf-1929a5d010fb is now active with session: e1d4f42e-2a31-4f5a-90fb-733be722b134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14731305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.14732499Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376,"session_id":"e1d4f42e-2a31-4f5a-90fb-733be722b134"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147337039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} {"msg":"Activated sub_volume 57a07404-f8db-4570-91cf-1929a5d010fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:40.147348259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 320 len 40960 data: 4 3 2 3 6 6 7 7 5 5 02/30 Read block 277 len 12288 03/30 Read block 96 len 12288 04/30 Flush 05/30 Write block 281 len 8192 data: 5 4 06/30 Read block 190 len 12288 07/30 Flush 08/30 Write block 334 len 20480 data: 5 5 5 5 6 09/30 Write block 238 len 20480 data: 5 6 6 5 4 10/30 Write block 351 len 20480 data: 5 4 4 4 4 11/30 Write block 443 len 16384 data: 3 2 2 3 12/30 Write block 169 len 28672 data: 6 6 5 5 6 5 7 13/30 Read block 18 len 20480 14/30 Read block 527 len 40960 15/30 Read block 397 len 24576 16/30 Read block 109 len 28672 17/30 Read block 291 len 40960 18/30 Read block 256 len 4096 19/30 Write block 482 len 32768 data: 5 6 6 6 6 5 3 3 20/30 Read block 115 len 12288 21/30 Read block 289 len 20480 22/30 Write block 326 len 8192 data: 8 8 23/30 Write block 501 len 36864 data: 5 6 7 6 6 7 7 6 6 24/30 Write block 502 len 24576 data: 7 8 7 7 8 8 25/30 Write block 384 len 20480 data: 4 5 5 3 3 26/30 Read block 250 len 4096 27/30 Read block 512 len 32768 28/30 Write block 263 len 4096 data: 4 29/30 Write block 262 len 36864 data: 4 5 4 5 6 6 7 6 7 30/30 Read block 258 len 24576 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 23 22 21 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:40.16723935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 44 44 44 74 74 74 F F F 8 160-179 44 44 44 74 74 74 T T T 9 180-199 37 37 37 63 63 63 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 45 45 45 77 77 77 T T T 12 240-259 44 44 44 76 76 76 T T T 13 260-279 44 44 44 76 76 76 T T T 14 280-299 46 46 46 79 79 79 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 46 46 46 78 78 78 T T T 17 340-359 44 44 44 76 76 76 T T T 18 360-379 44 44 44 76 76 76 F F F 19 380-399 44 44 44 76 76 76 T T T 20 400-419 44 44 44 76 76 76 F F F 21 420-439 44 44 44 76 76 76 F F F 22 440-459 38 38 38 69 69 69 T T T 23 460-479 30 30 30 50 50 50 F F F 24 480-499 42 42 42 73 73 73 T T T 25 500-519 42 42 42 73 73 73 T T T 26 520-539 41 41 41 72 72 72 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 46, Max flush: 79 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 39 39 39 70 70 70 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 40 40 40 71 71 71 F F F 7 140-159 44 44 44 74 74 74 F F F 8 160-179 47 47 47 80 80 80 F F F 9 180-199 37 37 37 63 63 63 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 47 47 47 80 80 80 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 47 47 47 80 80 80 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 47 47 47 80 80 80 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 44 44 44 76 76 76 F F F 21 420-439 44 44 44 76 76 76 F F F 22 440-459 47 47 47 80 80 80 F F F 23 460-479 30 30 30 50 50 50 F F F 24 480-499 47 47 47 80 80 80 F F F 25 500-519 47 47 47 80 80 80 F F F 26 520-539 41 41 41 72 72 72 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 47, Max flush: 80 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.426689572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.470063817Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.470646842Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.470688922Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.470704422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.470719162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} {"msg":"Crucible 3ea4574b-da4b-4a8e-8afb-e236caeac35a has session id: cb8e62ff-7aee-46fb-9e3e-7905abfed2ee","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.471297998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Upstairs opts: Upstairs UUID: 3ea4574b-da4b-4a8e-8afb-e236caeac35a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.471338327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Crucible stats registered with UUID: 3ea4574b-da4b-4a8e-8afb-e236caeac35a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.471369077Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:54.471393127Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.471867173Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.471900413Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} {"msg":"3ea4574b-da4b-4a8e-8afb-e236caeac35a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.471957823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.471984643Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"0","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472013692Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"0","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472028762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"1","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472043212Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"1","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472056422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"2","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472070282Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"2","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472091142Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472117531Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472142231Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:40335)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472172071Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:36693)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472201941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:41525)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.47234775Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472779877Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"1","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.472942315Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"0","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.473339972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"1","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.473450461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"0","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.473848298Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"2","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.474019297Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.474038357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.474436944Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"2","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[0]R flush_numbers[0..12]: [67, 76, 70, 73, 77, 71, 71, 74, 80, 63, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.47496863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[0]R generation[0..12]: [38, 44, 39, 42, 45, 40, 40, 44, 47, 37, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.47498863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.47500435Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[1]R flush_numbers[0..12]: [67, 76, 70, 73, 77, 71, 71, 74, 80, 63, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.4750183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[1]R generation[0..12]: [38, 44, 39, 42, 45, 40, 40, 44, 47, 37, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.47503917Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475064869Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[2]R flush_numbers[0..12]: [67, 76, 70, 73, 77, 71, 71, 74, 80, 63, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475092629Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[2]R generation[0..12]: [38, 44, 39, 42, 45, 40, 40, 44, 47, 37, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475116849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475143779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475174769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475200028Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475224038Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475248108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"":"downstairs","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475263148Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475283098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"0","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475307778Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"1","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475333527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"client":"2","":"downstairs","session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475349517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475365027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"3ea4574b-da4b-4a8e-8afb-e236caeac35a is now active with session: cb8e62ff-7aee-46fb-9e3e-7905abfed2ee","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:01:54.475392917Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475415447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384,"session_id":"cb8e62ff-7aee-46fb-9e3e-7905abfed2ee"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475435867Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} {"msg":"Activated sub_volume 3ea4574b-da4b-4a8e-8afb-e236caeac35a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:54.475455526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 192 len 20480 data: 3 3 4 4 4 02/30 Read block 532 len 12288 03/30 Write block 321 len 40960 data: 4 3 4 7 7 9 9 6 6 6 04/30 Read block 395 len 12288 05/30 Read block 240 len 40960 06/30 Write block 178 len 8192 data: 5 5 07/30 Write block 123 len 20480 data: 6 7 6 4 4 08/30 Write block 473 len 40960 data: 3 3 2 2 2 2 2 2 3 6 09/30 Write block 497 len 20480 data: 6 7 7 7 6 10/30 Read block 267 len 12288 11/30 Read block 193 len 4096 12/30 Read block 260 len 4096 13/30 Read block 384 len 16384 14/30 Read block 377 len 40960 15/30 Write block 51 len 4096 data: 4 16/30 Read block 566 len 8192 17/30 Read block 594 len 20480 18/30 Flush 19/30 Write block 132 len 4096 data: 7 20/30 Read block 590 len 20480 21/30 Write block 403 len 8192 data: 10 10 22/30 Write block 54 len 24576 data: 5 8 8 6 6 5 23/30 Read block 95 len 40960 24/30 Read block 500 len 16384 25/30 Read block 593 len 4096 26/30 Write block 321 len 4096 data: 5 27/30 Write block 275 len 8192 data: 5 3 28/30 Read block 237 len 36864 29/30 Write block 486 len 40960 data: 7 6 4 4 4 6 6 6 6 6 30/30 Write block 485 len 32768 data: 7 8 7 5 5 5 7 7 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:54.494185396Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 48 48 48 81 81 81 T T T 3 060-079 42 42 42 73 73 73 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 48 48 48 81 81 81 T T T 7 140-159 44 44 44 74 74 74 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 47 47 47 80 80 80 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 47 47 47 80 80 80 T T T 14 280-299 46 46 46 79 79 79 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 48 48 48 81 81 81 T T T 17 340-359 47 47 47 80 80 80 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 44 44 44 76 76 76 T T T 21 420-439 44 44 44 76 76 76 F F F 22 440-459 47 47 47 80 80 80 F F F 23 460-479 48 48 48 81 81 81 F F F 24 480-499 48 48 48 81 81 81 T T T 25 500-519 48 48 48 81 81 81 F F F 26 520-539 41 41 41 72 72 72 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 48, Max flush: 81 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 42 42 42 73 73 73 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 49 49 49 82 82 82 F F F 7 140-159 44 44 44 74 74 74 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 47 47 47 80 80 80 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 44 44 44 76 76 76 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 44 44 44 76 76 76 F F F 22 440-459 47 47 47 80 80 80 F F F 23 460-479 48 48 48 81 81 81 F F F 24 480-499 49 49 49 82 82 82 F F F 25 500-519 48 48 48 81 81 81 F F F 26 520-539 41 41 41 72 72 72 F F F 27 540-559 44 44 44 74 74 74 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 41 41 41 72 72 72 F F F Max gen: 49, Max flush: 82 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.730136927Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.773428693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.774003308Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.774051678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.774068578Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.774083368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} {"msg":"Crucible 3f0ac1e3-499e-4d4a-a626-8f226507ef7f has session id: 1ad22f22-88b0-497b-9330-fa632621d2ac","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.774601784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Upstairs opts: Upstairs UUID: 3f0ac1e3-499e-4d4a-a626-8f226507ef7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.774642823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Crucible stats registered with UUID: 3f0ac1e3-499e-4d4a-a626-8f226507ef7f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.774666563Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:58.774687133Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.77510841Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775148589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} {"msg":"3f0ac1e3-499e-4d4a-a626-8f226507ef7f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775204659Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775231049Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"0","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775271149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"0","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775300039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"1","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775330268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"1","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775360048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"2","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775434867Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"2","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775466177Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775493957Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775521657Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:45164)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775549746Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:37261)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775577836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:52721)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.775661596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.776117472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"0","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.776143522Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"1","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.776797787Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"1","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.776957976Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"0","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.777356933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.777462392Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.777494512Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"2","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778026538Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"2","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[0]R flush_numbers[0..12]: [67, 76, 82, 73, 77, 71, 82, 74, 81, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778515134Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[0]R generation[0..12]: [38, 44, 49, 42, 45, 40, 49, 44, 48, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778545684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778564994Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[1]R flush_numbers[0..12]: [67, 76, 82, 73, 77, 71, 82, 74, 81, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778582704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[1]R generation[0..12]: [38, 44, 49, 42, 45, 40, 49, 44, 48, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778600614Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778618023Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[2]R flush_numbers[0..12]: [67, 76, 82, 73, 77, 71, 82, 74, 81, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778634903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[2]R generation[0..12]: [38, 44, 49, 42, 45, 40, 49, 44, 48, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778651713Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778668633Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:58.778684373Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778705743Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778721613Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778742793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"":"downstairs","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778759333Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778773853Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"0","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778790032Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"1","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778806312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"client":"2","":"downstairs","session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778822302Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"level":30,"time":"2025-03-18T05:01:58.778841012Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"3f0ac1e3-499e-4d4a-a626-8f226507ef7f is now active with session: 1ad22f22-88b0-497b-9330-fa632621d2ac","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778857112Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778871922Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392,"session_id":"1ad22f22-88b0-497b-9330-fa632621d2ac"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778886861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} {"msg":"Activated sub_volume 3f0ac1e3-499e-4d4a-a626-8f226507ef7f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.778900952Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 552 len 24576 data: 4 4 4 6 7 9 02/30 Flush 03/30 Flush 04/30 Write block 484 len 24576 data: 7 8 9 8 6 6 05/30 Read block 439 len 28672 06/30 Write block 520 len 32768 data: 5 8 8 7 6 6 7 6 07/30 Read block 386 len 40960 08/30 Write block 78 len 8192 data: 6 6 09/30 Write block 285 len 40960 data: 7 6 7 7 6 6 6 8 7 5 10/30 Flush 11/30 Write block 372 len 12288 data: 6 6 7 12/30 Read block 403 len 4096 13/30 Write block 582 len 20480 data: 3 3 3 2 2 14/30 Read block 261 len 12288 15/30 Flush 16/30 Write block 541 len 24576 data: 5 5 4 4 5 6 17/30 Write block 136 len 8192 data: 5 5 18/30 Write block 438 len 28672 data: 7 7 5 3 3 4 3 19/30 Write block 590 len 8192 data: 6 5 20/30 Read block 457 len 16384 21/30 Read block 464 len 40960 22/30 Flush 23/30 Write block 517 len 32768 data: 3 4 6 6 9 9 8 7 24/30 Read block 520 len 20480 25/30 Write block 499 len 4096 data: 8 26/30 Write block 541 len 36864 data: 6 6 5 5 6 7 8 6 6 27/30 Read block 564 len 36864 28/30 Read block 539 len 40960 29/30 Write block 365 len 16384 data: 3 4 5 5 30/30 Read block 143 len 20480 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:58.799521957Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 44 44 44 74 74 74 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 47 47 47 80 80 80 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 50 50 50 85 85 85 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 50 50 50 86 86 86 T T T 19 380-399 47 47 47 80 80 80 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 50 50 50 87 87 87 F F F 22 440-459 50 50 50 87 87 87 F F F 23 460-479 48 48 48 81 81 81 F F F 24 480-499 50 50 50 85 85 85 T T T 25 500-519 48 48 48 81 81 81 T T T 26 520-539 50 50 50 85 85 85 T T T 27 540-559 50 50 50 87 87 87 T T T 28 560-579 44 44 44 74 74 74 F F F 29 580-599 50 50 50 87 87 87 F F F Max gen: 50, Max flush: 87 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 67 67 67 F F F 1 020-039 44 44 44 76 76 76 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 44 44 44 74 74 74 F F F 8 160-179 48 48 48 81 81 81 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 47 47 47 80 80 80 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 50 50 50 85 85 85 F F F 15 300-319 41 41 41 72 72 72 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 50 50 50 87 87 87 F F F 22 440-459 50 50 50 87 87 87 F F F 23 460-479 48 48 48 81 81 81 F F F 24 480-499 51 51 51 88 88 88 F F F 25 500-519 51 51 51 88 88 88 F F F 26 520-539 51 51 51 88 88 88 F F F 27 540-559 51 51 51 88 88 88 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 50 50 50 87 87 87 F F F Max gen: 51, Max flush: 88 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.037559461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:03.079766454Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.080123132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.111497057Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.112061873Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.112097303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.112115192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.112132822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} {"msg":"Crucible 86ac12c3-3e9f-4973-af50-fcb6a0da06cc has session id: 65504bd0-b15f-4643-840e-35e1444ab53f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.112764158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Upstairs opts: Upstairs UUID: 86ac12c3-3e9f-4973-af50-fcb6a0da06cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.112803437Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Crucible stats registered with UUID: 86ac12c3-3e9f-4973-af50-fcb6a0da06cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.112823637Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:03.112840277Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113208464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113231064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} {"msg":"86ac12c3-3e9f-4973-af50-fcb6a0da06cc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113297894Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113315893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"0","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113350473Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"0","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113376653Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"1","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113433793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"1","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113466072Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"2","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113493152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"2","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113510722Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113524532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113537832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"ds_connection connected from Ok(127.0.0.1:36813)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113551601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50033)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.113565201Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50176)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.11375579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.114099968Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"2","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.114268686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"1","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.114442275Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"0","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.114825842Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"1","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.114851452Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"2","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.11513745Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"0","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115407178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115488557Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[0]R flush_numbers[0..12]: [67, 76, 82, 85, 77, 71, 87, 74, 81, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115686096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[0]R generation[0..12]: [38, 44, 49, 50, 45, 40, 50, 44, 48, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115707576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115722405Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[1]R flush_numbers[0..12]: [67, 76, 82, 85, 77, 71, 87, 74, 81, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115735975Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[1]R generation[0..12]: [38, 44, 49, 50, 45, 40, 50, 44, 48, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115749385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115763335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[2]R flush_numbers[0..12]: [67, 76, 82, 85, 77, 71, 87, 74, 81, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115781425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[2]R generation[0..12]: [38, 44, 49, 50, 45, 40, 50, 44, 48, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115795415Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115809305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115821975Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115834804Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115847484Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115860074Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"":"downstairs","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115873064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115885034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"0","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115898284Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"1","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115911494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"client":"2","":"downstairs","session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:02:03.115924604Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115943044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"86ac12c3-3e9f-4973-af50-fcb6a0da06cc is now active with session: 65504bd0-b15f-4643-840e-35e1444ab53f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115955954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115967734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400,"session_id":"65504bd0-b15f-4643-840e-35e1444ab53f"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115981683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} {"msg":"Activated sub_volume 86ac12c3-3e9f-4973-af50-fcb6a0da06cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:03.115993303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 321 len 32768 data: 6 4 5 8 8 10 10 7 02/30 Write block 522 len 4096 data: 10 03/30 Write block 309 len 20480 data: 2 2 2 3 3 04/30 Read block 398 len 24576 05/30 Read block 150 len 32768 06/30 Read block 278 len 32768 07/30 Read block 333 len 32768 08/30 Read block 54 len 36864 09/30 Write block 584 len 20480 data: 4 3 3 3 4 10/30 Write block 299 len 40960 data: 4 6 6 5 4 4 4 3 3 2 11/30 Write block 285 len 28672 data: 8 7 8 8 7 7 7 12/30 Write block 7 len 24576 data: 6 6 8 7 7 6 13/30 Write block 24 len 20480 data: 6 6 8 6 5 14/30 Write block 158 len 40960 data: 7 6 6 5 5 5 4 6 8 7 15/30 Read block 37 len 32768 16/30 Read block 96 len 20480 17/30 Flush 18/30 Flush 19/30 Write block 48 len 8192 data: 3 3 20/30 Read block 426 len 28672 21/30 Write block 552 len 8192 data: 5 5 22/30 Read block 183 len 4096 23/30 Read block 386 len 20480 24/30 Flush 25/30 Write block 462 len 8192 data: 5 5 26/30 Flush 27/30 Write block 507 len 20480 data: 9 7 7 7 6 28/30 Read block 161 len 12288 29/30 Read block 184 len 24576 30/30 Write block 411 len 24576 data: 5 4 4 3 3 2 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:03.135795285Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 89 89 89 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 91 91 91 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 52 52 52 89 89 89 F F F 8 160-179 52 52 52 89 89 89 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 47 47 47 80 80 80 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 52 52 52 89 89 89 F F F 15 300-319 52 52 52 89 89 89 F F F 16 320-339 52 52 52 89 89 89 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 49 49 49 82 82 82 T T T 21 420-439 50 50 50 87 87 87 F F F 22 440-459 50 50 50 87 87 87 F F F 23 460-479 52 52 52 92 92 92 F F F 24 480-499 51 51 51 88 88 88 F F F 25 500-519 51 51 51 88 88 88 T T T 26 520-539 52 52 52 89 89 89 F F F 27 540-559 52 52 52 91 91 91 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 52, Max flush: 92 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 89 89 89 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 91 91 91 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 52 52 52 89 89 89 F F F 8 160-179 52 52 52 89 89 89 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 45 45 45 77 77 77 F F F 11 220-239 47 47 47 80 80 80 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 52 52 52 89 89 89 F F F 15 300-319 52 52 52 89 89 89 F F F 16 320-339 52 52 52 89 89 89 F F F 17 340-359 47 47 47 80 80 80 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 53 53 53 93 93 93 F F F 21 420-439 50 50 50 87 87 87 F F F 22 440-459 50 50 50 87 87 87 F F F 23 460-479 52 52 52 92 92 92 F F F 24 480-499 51 51 51 88 88 88 F F F 25 500-519 53 53 53 93 93 93 F F F 26 520-539 52 52 52 89 89 89 F F F 27 540-559 52 52 52 91 91 91 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 53, Max flush: 93 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.388011243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.432173893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.432755778Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.432799358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.432815038Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.432829318Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} {"msg":"Crucible fe4041c9-f3da-4fe1-bb68-6ebacd9973c1 has session id: e5876b88-9496-453c-b745-7923703d090d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.433283064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Upstairs opts: Upstairs UUID: fe4041c9-f3da-4fe1-bb68-6ebacd9973c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.433324234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Crucible stats registered with UUID: fe4041c9-f3da-4fe1-bb68-6ebacd9973c1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.433347954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:07.433362864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.43384487Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.43387395Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} {"msg":"fe4041c9-f3da-4fe1-bb68-6ebacd9973c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.433955669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.433972809Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"0","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434000399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"0","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434014829Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"1","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434029259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"1","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434042208Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"2","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434056179Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"2","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434069408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434088638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434119278Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52094)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434147558Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65114)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434172067Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57831)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434195817Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.434920522Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"2","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.435086321Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"0","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.435468308Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"1","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.435556937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"2","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.435791675Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"0","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436093863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436178132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"1","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436292662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[0]R flush_numbers[0..12]: [89, 89, 91, 85, 77, 71, 87, 89, 89, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436691089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[0]R generation[0..12]: [52, 52, 52, 50, 45, 40, 50, 52, 52, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436715788Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436736198Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[1]R flush_numbers[0..12]: [89, 89, 91, 85, 77, 71, 87, 89, 89, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436751528Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[1]R generation[0..12]: [52, 52, 52, 50, 45, 40, 50, 52, 52, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436765398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436779378Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[2]R flush_numbers[0..12]: [89, 89, 91, 85, 77, 71, 87, 89, 89, 81, 77, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436793038Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[2]R generation[0..12]: [52, 52, 52, 50, 45, 40, 50, 52, 52, 48, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436806438Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436820378Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436838878Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436869207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436887047Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436909807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"":"downstairs","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436934767Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436955557Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"0","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.436979246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"1","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.437023476Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"client":"2","":"downstairs","session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.437057566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.437082506Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"fe4041c9-f3da-4fe1-bb68-6ebacd9973c1 is now active with session: e5876b88-9496-453c-b745-7923703d090d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.437103956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.437116865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408,"session_id":"e5876b88-9496-453c-b745-7923703d090d"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.437129085Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} {"msg":"Activated sub_volume fe4041c9-f3da-4fe1-bb68-6ebacd9973c1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:07.437140285Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 225 len 28672 data: 4 4 5 7 5 6 4 02/30 Read block 443 len 32768 03/30 Flush 04/30 Write block 488 len 32768 data: 7 7 6 8 8 7 7 7 05/30 Read block 457 len 36864 06/30 Read block 38 len 36864 07/30 Write block 457 len 36864 data: 4 2 3 4 5 6 6 3 3 08/30 Flush 09/30 Write block 416 len 12288 data: 3 2 2 10/30 Read block 124 len 12288 11/30 Write block 14 len 24576 data: 7 7 5 5 4 4 12/30 Write block 429 len 32768 data: 5 6 10 9 8 8 8 6 13/30 Read block 138 len 24576 14/30 Write block 330 len 36864 data: 7 6 6 5 6 6 6 6 7 15/30 Read block 293 len 24576 16/30 Write block 167 len 4096 data: 8 17/30 Write block 224 len 36864 data: 4 5 5 6 8 6 7 5 5 18/30 Flush 19/30 Write block 354 len 4096 data: 5 20/30 Write block 530 len 20480 data: 4 3 3 3 3 21/30 Read block 365 len 24576 22/30 Read block 577 len 12288 23/30 Read block 238 len 8192 24/30 Read block 558 len 28672 25/30 Read block 19 len 4096 26/30 Read block 89 len 24576 27/30 Read block 478 len 4096 28/30 Write block 211 len 36864 data: 5 4 4 5 5 6 6 4 4 29/30 Read block 502 len 4096 30/30 Read block 298 len 40960 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 12 12 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:07.458608595Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 96 96 96 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 91 91 91 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 52 52 52 89 89 89 F F F 8 160-179 54 54 54 96 96 96 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 45 45 45 77 77 77 T T T 11 220-239 54 54 54 96 96 96 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 52 52 52 89 89 89 F F F 15 300-319 52 52 52 89 89 89 F F F 16 320-339 54 54 54 96 96 96 F F F 17 340-359 47 47 47 80 80 80 T T T 18 360-379 51 51 51 88 88 88 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 54 54 54 96 96 96 F F F 21 420-439 54 54 54 96 96 96 F F F 22 440-459 54 54 54 95 95 95 F F F 23 460-479 54 54 54 95 95 95 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 53 53 53 93 93 93 F F F 26 520-539 52 52 52 89 89 89 T T T 27 540-559 52 52 52 91 91 91 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 54, Max flush: 96 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 96 96 96 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 52 52 52 91 91 91 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 52 52 52 89 89 89 F F F 8 160-179 54 54 54 96 96 96 F F F 9 180-199 48 48 48 81 81 81 F F F 10 200-219 55 55 55 97 97 97 F F F 11 220-239 54 54 54 96 96 96 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 52 52 52 89 89 89 F F F 15 300-319 52 52 52 89 89 89 F F F 16 320-339 54 54 54 96 96 96 F F F 17 340-359 55 55 55 97 97 97 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 54 54 54 96 96 96 F F F 21 420-439 54 54 54 96 96 96 F F F 22 440-459 54 54 54 95 95 95 F F F 23 460-479 54 54 54 95 95 95 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 53 53 53 93 93 93 F F F 26 520-539 55 55 55 97 97 97 F F F 27 540-559 52 52 52 91 91 91 F F F 28 560-579 44 44 44 74 74 74 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 55, Max flush: 97 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.684425212Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.727827837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.728387803Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.728436822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.728456362Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.728474102Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} {"msg":"Crucible c13e97ad-ba9d-49ab-a1d7-2d53886d9044 has session id: a2fe055e-6048-48f0-aed7-dbc542e8e6d4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729027108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Upstairs opts: Upstairs UUID: c13e97ad-ba9d-49ab-a1d7-2d53886d9044, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729066818Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Crucible stats registered with UUID: c13e97ad-ba9d-49ab-a1d7-2d53886d9044","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729090047Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:11.729110987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729460504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729493454Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} {"msg":"c13e97ad-ba9d-49ab-a1d7-2d53886d9044 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729547624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729575824Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"0","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729620843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"0","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729658433Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"1","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729688963Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"1","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729711183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"2","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729737062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"2","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729762982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729790272Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729818212Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62365)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729910731Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35245)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729954061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:59892)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.729983141Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.730702565Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"0","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.730734755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"1","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.731273601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"1","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.73140207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"0","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.731511969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"2","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.731856837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.731984456Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732268514Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"2","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[0]R flush_numbers[0..12]: [96, 89, 91, 85, 77, 71, 87, 89, 96, 81, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.73277349Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[0]R generation[0..12]: [54, 52, 52, 50, 45, 40, 50, 52, 54, 48, 55, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.73279476Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.73280967Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[1]R flush_numbers[0..12]: [96, 89, 91, 85, 77, 71, 87, 89, 96, 81, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732823469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[1]R generation[0..12]: [54, 52, 52, 50, 45, 40, 50, 52, 54, 48, 55, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732837469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732851629Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[2]R flush_numbers[0..12]: [96, 89, 91, 85, 77, 71, 87, 89, 96, 81, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732864999Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[2]R generation[0..12]: [54, 52, 52, 50, 45, 40, 50, 52, 54, 48, 55, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732878459Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732892049Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732904809Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732921999Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732936218Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732949078Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"":"downstairs","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732970108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732982028Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"0","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.732995138Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"1","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.733008078Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"client":"2","":"downstairs","session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.733021148Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.733033188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"c13e97ad-ba9d-49ab-a1d7-2d53886d9044 is now active with session: a2fe055e-6048-48f0-aed7-dbc542e8e6d4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.733045728Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.733057398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416,"session_id":"a2fe055e-6048-48f0-aed7-dbc542e8e6d4"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.733069448Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} {"msg":"Activated sub_volume c13e97ad-ba9d-49ab-a1d7-2d53886d9044","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:11.733080608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 284 len 8192 data: 6 9 02/30 Write block 409 len 36864 data: 7 6 6 5 5 4 4 4 3 03/30 Read block 586 len 32768 04/30 Read block 61 len 36864 05/30 Write block 421 len 20480 data: 6 6 6 8 6 06/30 Write block 185 len 36864 data: 5 3 3 4 5 4 4 4 4 07/30 Read block 23 len 24576 08/30 Write block 337 len 16384 data: 7 8 6 5 09/30 Write block 38 len 40960 data: 8 8 6 5 3 4 5 4 4 4 10/30 Read block 12 len 36864 11/30 Write block 104 len 24576 data: 3 3 3 3 3 3 12/30 Write block 157 len 20480 data: 4 8 7 7 6 13/30 Write block 570 len 8192 data: 7 5 14/30 Write block 547 len 24576 data: 9 7 7 6 3 6 15/30 Read block 327 len 16384 16/30 Write block 190 len 32768 data: 5 5 5 5 5 5 5 5 17/30 Write block 465 len 16384 data: 4 3 3 4 18/30 Write block 165 len 24576 data: 7 9 9 6 7 7 19/30 Read block 217 len 16384 20/30 Read block 535 len 24576 21/30 Write block 468 len 16384 data: 5 4 4 4 22/30 Read block 547 len 4096 23/30 Read block 37 len 32768 24/30 Write block 11 len 28672 data: 8 7 6 8 8 6 6 25/30 Write block 518 len 36864 data: 5 7 7 10 11 9 8 7 8 26/30 Write block 199 len 40960 data: 4 4 4 3 3 3 4 3 3 4 27/30 Read block 253 len 12288 28/30 Write block 460 len 32768 data: 5 6 7 7 4 5 4 4 29/30 Read block 486 len 28672 30/30 Write block 311 len 28672 data: 3 4 4 4 5 5 4 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:11.753531464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 96 96 96 T T T 1 020-039 52 52 52 89 89 89 T T T 2 040-059 52 52 52 91 91 91 T T T 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 40 40 40 71 71 71 T T T 6 120-139 50 50 50 87 87 87 F F F 7 140-159 52 52 52 89 89 89 T T T 8 160-179 54 54 54 96 96 96 T T T 9 180-199 48 48 48 81 81 81 T T T 10 200-219 55 55 55 97 97 97 T T T 11 220-239 54 54 54 96 96 96 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 52 52 52 89 89 89 T T T 15 300-319 52 52 52 89 89 89 T T T 16 320-339 54 54 54 96 96 96 T T T 17 340-359 55 55 55 97 97 97 T T T 18 360-379 51 51 51 88 88 88 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 54 54 54 96 96 96 T T T 21 420-439 54 54 54 96 96 96 T T T 22 440-459 54 54 54 95 95 95 F F F 23 460-479 54 54 54 95 95 95 T T T 24 480-499 54 54 54 95 95 95 F F F 25 500-519 53 53 53 93 93 93 T T T 26 520-539 55 55 55 97 97 97 T T T 27 540-559 52 52 52 91 91 91 T T T 28 560-579 44 44 44 74 74 74 T T T 29 580-599 52 52 52 89 89 89 F F F Max gen: 55, Max flush: 97 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 98 98 98 F F F 1 020-039 56 56 56 98 98 98 F F F 2 040-059 56 56 56 98 98 98 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 45 45 45 77 77 77 F F F 5 100-119 56 56 56 98 98 98 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 56 56 56 98 98 98 F F F 9 180-199 56 56 56 98 98 98 F F F 10 200-219 56 56 56 98 98 98 F F F 11 220-239 54 54 54 96 96 96 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 F F F 14 280-299 56 56 56 98 98 98 F F F 15 300-319 56 56 56 98 98 98 F F F 16 320-339 56 56 56 98 98 98 F F F 17 340-359 56 56 56 98 98 98 F F F 18 360-379 51 51 51 88 88 88 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 56 56 56 98 98 98 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 54 54 54 95 95 95 F F F 23 460-479 56 56 56 98 98 98 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 56 56 56 98 98 98 F F F 27 540-559 56 56 56 98 98 98 F F F 28 560-579 56 56 56 98 98 98 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 56, Max flush: 98 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.065318725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.109218776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.109790492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.109841391Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.109864761Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.109883311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} {"msg":"Crucible 19d74fa9-23f0-4d08-89c9-37ec19ae7d1d has session id: 179ca12a-d042-4ec6-8578-0aa67c5d1b95","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.110519736Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Upstairs opts: Upstairs UUID: 19d74fa9-23f0-4d08-89c9-37ec19ae7d1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.110562546Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Crucible stats registered with UUID: 19d74fa9-23f0-4d08-89c9-37ec19ae7d1d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.110588556Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:26.110609616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111165191Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111190181Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} {"msg":"19d74fa9-23f0-4d08-89c9-37ec19ae7d1d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111261941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111290561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"0","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11132868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"0","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11135399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"1","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111439469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"1","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111470519Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"2","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111496589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"2","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111524489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111550819Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111576518Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"ds_connection connected from Ok(127.0.0.1:55449)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111604288Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"ds_connection connected from Ok(127.0.0.1:61555)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111719537Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"ds_connection connected from Ok(127.0.0.1:50201)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.111744937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.112231063Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"1","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.112264703Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"0","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.112563471Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"2","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113043027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"1","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113071517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"0","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113144947Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"2","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113652313Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113761842Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[0]R flush_numbers[0..12]: [98, 98, 98, 85, 77, 98, 87, 98, 98, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113827542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[0]R generation[0..12]: [56, 56, 56, 50, 45, 56, 50, 56, 56, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113852091Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113867911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[1]R flush_numbers[0..12]: [98, 98, 98, 85, 77, 98, 87, 98, 98, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113881651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[1]R generation[0..12]: [56, 56, 56, 50, 45, 56, 50, 56, 56, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113895031Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113908721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[2]R flush_numbers[0..12]: [98, 98, 98, 85, 77, 98, 87, 98, 98, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113922151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[2]R generation[0..12]: [56, 56, 56, 50, 45, 56, 50, 56, 56, 56, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113935271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113949121Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Max found gen is 57","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.113961901Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Generation requested: 58 >= found:57","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11397485Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11398764Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11400039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"":"downstairs","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11401353Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11402619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"0","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11404185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"1","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11405518Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"client":"2","":"downstairs","session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11406827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:02:26.11408048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"19d74fa9-23f0-4d08-89c9-37ec19ae7d1d is now active with session: 179ca12a-d042-4ec6-8578-0aa67c5d1b95","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11409732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.11410954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424,"session_id":"179ca12a-d042-4ec6-8578-0aa67c5d1b95"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.114122019Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} {"msg":"Activated sub_volume 19d74fa9-23f0-4d08-89c9-37ec19ae7d1d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.114133319Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 49 len 8192 data: 4 3 02/30 Write block 313 len 36864 data: 5 5 6 6 5 4 4 5 7 03/30 Write block 566 len 4096 data: 5 04/30 Read block 517 len 40960 05/30 Read block 147 len 28672 06/30 Write block 61 len 4096 data: 5 07/30 Read block 14 len 20480 08/30 Read block 221 len 16384 09/30 Read block 560 len 28672 10/30 Read block 34 len 16384 11/30 Write block 235 len 4096 data: 6 12/30 Flush 13/30 Write block 345 len 40960 data: 3 2 3 2 4 4 6 5 5 6 14/30 Write block 360 len 8192 data: 3 4 15/30 Write block 308 len 20480 data: 3 3 3 4 5 16/30 Read block 199 len 8192 17/30 Flush 18/30 Write block 92 len 16384 data: 5 5 5 5 19/30 Write block 278 len 12288 data: 4 4 4 20/30 Read block 157 len 24576 21/30 Read block 274 len 32768 22/30 Read block 347 len 32768 23/30 Write block 353 len 16384 data: 6 7 5 4 24/30 Write block 176 len 36864 data: 6 6 6 6 5 5 6 5 5 25/30 Read block 582 len 28672 26/30 Write block 91 len 40960 data: 5 6 6 6 6 5 4 3 3 3 27/30 Write block 293 len 28672 data: 8 6 3 3 3 3 5 28/30 Read block 211 len 12288 29/30 Write block 409 len 20480 data: 8 7 7 6 6 30/30 Read block 219 len 36864 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 12 13 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:26.133710923Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 98 98 98 F F F 1 020-039 56 56 56 98 98 98 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 45 45 45 77 77 77 T T T 5 100-119 56 56 56 98 98 98 T T T 6 120-139 50 50 50 87 87 87 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 56 56 56 98 98 98 T T T 9 180-199 56 56 56 98 98 98 T T T 10 200-219 56 56 56 98 98 98 F F F 11 220-239 58 58 58 99 99 99 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 49 49 49 82 82 82 T T T 14 280-299 56 56 56 98 98 98 T T T 15 300-319 58 58 58 100 100 100 F F F 16 320-339 58 58 58 99 99 99 F F F 17 340-359 58 58 58 100 100 100 T T T 18 360-379 58 58 58 100 100 100 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 56 56 56 98 98 98 T T T 21 420-439 56 56 56 98 98 98 F F F 22 440-459 54 54 54 95 95 95 F F F 23 460-479 56 56 56 98 98 98 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 56 56 56 98 98 98 F F F 27 540-559 56 56 56 98 98 98 F F F 28 560-579 58 58 58 99 99 99 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 58, Max flush: 100 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 98 98 98 F F F 1 020-039 56 56 56 98 98 98 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 50 50 50 87 87 87 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 59 59 59 101 101 101 F F F 9 180-199 59 59 59 101 101 101 F F F 10 200-219 56 56 56 98 98 98 F F F 11 220-239 58 58 58 99 99 99 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 59 59 59 101 101 101 F F F 15 300-319 58 58 58 100 100 100 F F F 16 320-339 58 58 58 99 99 99 F F F 17 340-359 59 59 59 101 101 101 F F F 18 360-379 58 58 58 100 100 100 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 59 59 59 101 101 101 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 54 54 54 95 95 95 F F F 23 460-479 56 56 56 98 98 98 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 56 56 56 98 98 98 F F F 27 540-559 56 56 56 98 98 98 F F F 28 560-579 58 58 58 99 99 99 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 59, Max flush: 101 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.382737245Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:30.425181547Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.425555574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.456519342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.457045938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.457081308Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.457095628Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.457111048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} {"msg":"Crucible 35fe3d71-a961-4afc-9fa5-5d158bdc2ef4 has session id: e9893a06-3470-4d02-9775-7c5b50e5474d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.457618734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Upstairs opts: Upstairs UUID: 35fe3d71-a961-4afc-9fa5-5d158bdc2ef4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.457684314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Crucible stats registered with UUID: 35fe3d71-a961-4afc-9fa5-5d158bdc2ef4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.457712803Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:30.457727243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.45820646Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.45823943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} {"msg":"35fe3d71-a961-4afc-9fa5-5d158bdc2ef4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458296849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458314759Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"0","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458342009Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"0","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458356229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"1","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458381748Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"1","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458407768Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"2","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458433998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"2","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458460868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458476648Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458490388Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41056)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458512907Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43701)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458537567Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34018)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.458697536Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.459248212Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"1","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.459282132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"2","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.459360801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"0","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.459884497Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"1","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.459962397Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"0","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.459982756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"2","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460430463Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460542402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[0]R flush_numbers[0..12]: [98, 98, 99, 99, 101, 101, 87, 98, 101, 101, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460624732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[0]R generation[0..12]: [56, 56, 58, 58, 59, 59, 50, 56, 59, 59, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460645591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460660301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[1]R flush_numbers[0..12]: [98, 98, 99, 99, 101, 101, 87, 98, 101, 101, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460674031Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[1]R generation[0..12]: [56, 56, 58, 58, 59, 59, 50, 56, 59, 59, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460687621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460702941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[2]R flush_numbers[0..12]: [98, 98, 99, 99, 101, 101, 87, 98, 101, 101, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460717651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[2]R generation[0..12]: [56, 56, 58, 58, 59, 59, 50, 56, 59, 59, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460734851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460749851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460763491Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46077652Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46078926Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:02:30.46080176Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"":"downstairs","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46082003Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46083186Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"0","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46084571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"1","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46085939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"client":"2","":"downstairs","session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46087235Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.46088431Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"35fe3d71-a961-4afc-9fa5-5d158bdc2ef4 is now active with session: e9893a06-3470-4d02-9775-7c5b50e5474d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460897679Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460909579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432,"session_id":"e9893a06-3470-4d02-9775-7c5b50e5474d"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460921779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} {"msg":"Activated sub_volume 35fe3d71-a961-4afc-9fa5-5d158bdc2ef4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:30.460933079Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 351 len 36864 data: 7 6 7 8 6 5 4 4 4 02/30 Write block 356 len 16384 data: 6 5 5 5 03/30 Write block 352 len 36864 data: 7 8 9 7 7 6 6 6 4 04/30 Read block 440 len 24576 05/30 Read block 245 len 16384 06/30 Write block 206 len 20480 data: 4 4 5 4 5 07/30 Read block 495 len 24576 08/30 Read block 32 len 32768 09/30 Read block 236 len 4096 10/30 Write block 228 len 12288 data: 9 7 8 11/30 Write block 322 len 28672 data: 5 6 9 9 11 11 8 12/30 Read block 569 len 20480 13/30 Flush 14/30 Write block 454 len 16384 data: 4 4 4 5 15/30 Read block 169 len 8192 16/30 Write block 265 len 40960 data: 6 7 7 8 7 8 8 8 7 5 17/30 Read block 118 len 24576 18/30 Write block 307 len 28672 data: 4 4 4 4 5 6 6 19/30 Write block 354 len 16384 data: 10 8 8 7 20/30 Flush 21/30 Write block 539 len 36864 data: 5 5 7 7 6 6 7 8 10 22/30 Write block 446 len 40960 data: 4 3 3 4 3 3 3 3 5 5 23/30 Read block 513 len 40960 24/30 Read block 506 len 20480 25/30 Write block 263 len 28672 data: 6 5 7 8 8 9 8 26/30 Write block 124 len 16384 data: 8 7 5 5 27/30 Read block 156 len 40960 28/30 Write block 19 len 36864 data: 5 3 4 4 7 7 7 9 7 29/30 Read block 392 len 20480 30/30 Write block 99 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:30.481784903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 98 98 98 T T T 1 020-039 56 56 56 98 98 98 T T T 2 040-059 58 58 58 99 99 99 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 59 59 59 101 101 101 T T T 5 100-119 59 59 59 101 101 101 T T T 6 120-139 50 50 50 87 87 87 T T T 7 140-159 56 56 56 98 98 98 F F F 8 160-179 59 59 59 101 101 101 F F F 9 180-199 59 59 59 101 101 101 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 60 60 60 103 103 103 T T T 14 280-299 59 59 59 101 101 101 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 60 60 60 102 102 102 F F F 17 340-359 60 60 60 103 103 103 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 59 59 59 101 101 101 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 60 60 60 103 103 103 T T T 23 460-479 56 56 56 98 98 98 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 56 56 56 98 98 98 T T T 27 540-559 56 56 56 98 98 98 T T T 28 560-579 58 58 58 99 99 99 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 60, Max flush: 103 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 104 104 104 F F F 1 020-039 61 61 61 104 104 104 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 61 61 61 104 104 104 F F F 5 100-119 61 61 61 104 104 104 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 59 59 59 101 101 101 F F F 9 180-199 59 59 59 101 101 101 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 59 59 59 101 101 101 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 60 60 60 102 102 102 F F F 17 340-359 60 60 60 103 103 103 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 47 47 47 80 80 80 F F F 20 400-419 59 59 59 101 101 101 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 56 56 56 98 98 98 F F F 24 480-499 54 54 54 95 95 95 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 61 61 61 104 104 104 F F F 27 540-559 61 61 61 104 104 104 F F F 28 560-579 58 58 58 99 99 99 F F F 29 580-599 52 52 52 89 89 89 F F F Max gen: 61, Max flush: 104 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.773339156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:34.816948299Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.817320057Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.848320015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.84887657Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.84891825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.84893671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.84895156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} {"msg":"Crucible 1fd9ba59-8f0e-40f4-9297-18c417e135c2 has session id: ceef9f07-d9e3-42ba-9616-b292174a1938","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.849491336Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Upstairs opts: Upstairs UUID: 1fd9ba59-8f0e-40f4-9297-18c417e135c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.849531515Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Crucible stats registered with UUID: 1fd9ba59-8f0e-40f4-9297-18c417e135c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.849555745Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:34.849577095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850060971Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850088031Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} {"msg":"1fd9ba59-8f0e-40f4-9297-18c417e135c2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850147841Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850173921Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"0","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850202381Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"0","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.85021658Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"1","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.85023147Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"1","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.85024492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"2","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850320089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"2","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850355769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850381799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850407759Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"ds_connection connected from Ok(127.0.0.1:49762)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850433219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"ds_connection connected from Ok(127.0.0.1:50510)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850647857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"ds_connection connected from Ok(127.0.0.1:49452)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.850688527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.851127763Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"1","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.851156243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"2","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.851171913Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"0","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.851822488Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"1","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.851850408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"2","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.851865378Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"0","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852490063Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852520213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[0]R flush_numbers[0..12]: [104, 104, 99, 99, 104, 104, 104, 98, 101, 101, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852563303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[0]R generation[0..12]: [61, 61, 58, 58, 61, 61, 61, 56, 59, 59, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852581583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852596003Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[1]R flush_numbers[0..12]: [104, 104, 99, 99, 104, 104, 104, 98, 101, 101, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852609783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[1]R generation[0..12]: [61, 61, 58, 58, 61, 61, 61, 56, 59, 59, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852623482Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852637342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[2]R flush_numbers[0..12]: [104, 104, 99, 99, 104, 104, 104, 98, 101, 101, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852650982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[2]R generation[0..12]: [61, 61, 58, 58, 61, 61, 61, 56, 59, 59, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852664462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852678132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852691242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852704572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852717442Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852730111Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"":"downstairs","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852743322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852757591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"0","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852771171Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"1","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-03-18T05:02:34.852793141Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"client":"2","":"downstairs","session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852806461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852818601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"1fd9ba59-8f0e-40f4-9297-18c417e135c2 is now active with session: ceef9f07-d9e3-42ba-9616-b292174a1938","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852831311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852843181Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440,"session_id":"ceef9f07-d9e3-42ba-9616-b292174a1938"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.852855401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} {"msg":"Activated sub_volume 1fd9ba59-8f0e-40f4-9297-18c417e135c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:34.85286665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 553 len 24576 data: 6 5 7 8 10 9 03/30 Write block 400 len 32768 data: 8 8 8 11 11 9 9 8 04/30 Read block 577 len 36864 05/30 Write block 399 len 4096 data: 5 06/30 Write block 179 len 36864 data: 7 6 6 7 6 6 6 4 4 07/30 Write block 543 len 8192 data: 7 7 08/30 Write block 15 len 36864 data: 9 7 7 5 6 4 5 5 8 09/30 Read block 103 len 28672 10/30 Write block 517 len 24576 data: 4 6 8 8 11 12 11/30 Read block 403 len 16384 12/30 Read block 477 len 24576 13/30 Write block 504 len 40960 data: 8 8 9 10 8 8 8 7 4 3 14/30 Write block 579 len 40960 data: 5 5 3 4 4 5 4 4 4 5 15/30 Write block 19 len 28672 data: 7 5 6 6 9 8 8 16/30 Read block 202 len 4096 17/30 Read block 185 len 12288 18/30 Write block 103 len 36864 data: 3 4 4 4 4 4 4 3 3 19/30 Read block 268 len 36864 20/30 Read block 383 len 28672 21/30 Write block 479 len 8192 data: 3 3 22/30 Write block 30 len 40960 data: 3 5 6 7 8 8 10 8 9 9 23/30 Read block 523 len 32768 24/30 Read block 119 len 24576 25/30 Read block 134 len 28672 26/30 Flush 27/30 Write block 592 len 28672 data: 6 4 4 5 4 4 4 28/30 Read block 545 len 12288 29/30 Write block 14 len 8192 data: 9 10 30/30 Read block 564 len 8192 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:34.874153991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 T T T 1 020-039 62 62 62 106 106 106 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 61 61 61 104 104 104 F F F 5 100-119 62 62 62 106 106 106 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 59 59 59 101 101 101 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 60 60 60 102 102 102 F F F 17 340-359 60 60 60 103 103 103 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 62 62 62 106 106 106 F F F 24 480-499 62 62 62 106 106 106 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 62 62 62 106 106 106 F F F 27 540-559 62 62 62 106 106 106 F F F 28 560-579 62 62 62 106 106 106 F F F 29 580-599 62 62 62 106 106 106 T T T Max gen: 62, Max flush: 106 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 107 107 107 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 61 61 61 104 104 104 F F F 5 100-119 62 62 62 106 106 106 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 59 59 59 101 101 101 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 60 60 60 102 102 102 F F F 17 340-359 60 60 60 103 103 103 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 62 62 62 106 106 106 F F F 24 480-499 62 62 62 106 106 106 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 62 62 62 106 106 106 F F F 27 540-559 62 62 62 106 106 106 F F F 28 560-579 62 62 62 106 106 106 F F F 29 580-599 63 63 63 107 107 107 F F F Max gen: 63, Max flush: 107 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.1289858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:39.171370923Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.17172355Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.202680408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.203243994Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.203278424Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.203293104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.203307303Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} {"msg":"Crucible ebe98062-209c-4616-8e90-9acc8980aa27 has session id: 6a213580-a824-4585-bede-53d813220c17","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.203887729Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Upstairs opts: Upstairs UUID: ebe98062-209c-4616-8e90-9acc8980aa27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.203926829Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Crucible stats registered with UUID: ebe98062-209c-4616-8e90-9acc8980aa27","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.203952428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:39.203973218Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204347325Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204381015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} {"msg":"ebe98062-209c-4616-8e90-9acc8980aa27 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204437425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204464965Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"0","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204495495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"0","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204510034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"1","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204525154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"1","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204538394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"2","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204552354Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"2","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204565504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204578874Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204592244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"ds_connection connected from Ok(127.0.0.1:48100)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204665463Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"ds_connection connected from Ok(127.0.0.1:56741)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204698993Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"ds_connection connected from Ok(127.0.0.1:49583)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.204726643Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.205455837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"2","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.205497367Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"1","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.205596006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"0","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206039433Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"1","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206080432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"2","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.2063678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"0","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206647168Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206749158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[0]R flush_numbers[0..12]: [107, 106, 99, 99, 104, 106, 104, 98, 106, 106, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206860427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[0]R generation[0..12]: [63, 62, 58, 58, 61, 62, 61, 56, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206881146Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206896626Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[1]R flush_numbers[0..12]: [107, 106, 99, 99, 104, 106, 104, 98, 106, 106, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206910626Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[1]R generation[0..12]: [63, 62, 58, 58, 61, 62, 61, 56, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206924246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206939136Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[2]R flush_numbers[0..12]: [107, 106, 99, 99, 104, 106, 104, 98, 106, 106, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206957066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[2]R generation[0..12]: [63, 62, 58, 58, 61, 62, 61, 56, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206971566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206985736Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.206998856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207012056Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207024776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207037515Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"":"downstairs","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207050455Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207062355Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"0","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207075475Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"1","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207096425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"client":"2","":"downstairs","session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207109525Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207121615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"ebe98062-209c-4616-8e90-9acc8980aa27 is now active with session: 6a213580-a824-4585-bede-53d813220c17","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207134205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207145915Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448,"session_id":"6a213580-a824-4585-bede-53d813220c17"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207157875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} {"msg":"Activated sub_volume ebe98062-209c-4616-8e90-9acc8980aa27","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:39.207172204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 449 len 36864 data: 5 4 4 4 4 6 6 5 6 02/30 Read block 299 len 24576 03/30 Read block 411 len 8192 04/30 Read block 254 len 8192 05/30 Read block 358 len 24576 06/30 Read block 523 len 16384 07/30 Read block 267 len 12288 08/30 Write block 587 len 28672 data: 5 6 6 7 6 7 5 09/30 Flush 10/30 Flush 11/30 Flush 12/30 Write block 63 len 20480 data: 4 4 3 3 3 13/30 Read block 177 len 20480 14/30 Read block 239 len 16384 15/30 Read block 477 len 24576 16/30 Flush 17/30 Write block 443 len 36864 data: 5 4 3 5 4 4 6 5 5 18/30 Write block 507 len 28672 data: 11 9 9 9 8 5 4 19/30 Flush 20/30 Write block 559 len 12288 data: 8 8 8 21/30 Read block 549 len 24576 22/30 Write block 483 len 4096 data: 7 23/30 Read block 311 len 32768 24/30 Write block 481 len 40960 data: 4 7 8 8 9 10 9 8 8 7 25/30 Read block 258 len 40960 26/30 Read block 22 len 28672 27/30 Write block 321 len 20480 data: 8 6 7 10 10 28/30 Write block 92 len 8192 data: 7 7 29/30 Write block 149 len 16384 data: 3 4 5 5 30/30 Read block 247 len 40960 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 3 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:39.2277861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 107 107 107 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 64 64 64 111 111 111 F F F 4 080-099 61 61 61 104 104 104 T T T 5 100-119 62 62 62 106 106 106 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 56 56 56 98 98 98 T T T 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 59 59 59 101 101 101 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 60 60 60 102 102 102 T T T 17 340-359 60 60 60 103 103 103 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 64 64 64 112 112 112 F F F 23 460-479 62 62 62 106 106 106 F F F 24 480-499 62 62 62 106 106 106 T T T 25 500-519 64 64 64 112 112 112 F F F 26 520-539 62 62 62 106 106 106 F F F 27 540-559 62 62 62 106 106 106 T T T 28 560-579 62 62 62 106 106 106 T T T 29 580-599 64 64 64 108 108 108 F F F Max gen: 64, Max flush: 112 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 107 107 107 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 64 64 64 111 111 111 F F F 4 080-099 65 65 65 113 113 113 F F F 5 100-119 62 62 62 106 106 106 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 65 65 65 113 113 113 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 60 60 60 102 102 102 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 59 59 59 101 101 101 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 65 65 65 113 113 113 F F F 17 340-359 60 60 60 103 103 103 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 64 64 64 112 112 112 F F F 23 460-479 62 62 62 106 106 106 F F F 24 480-499 65 65 65 113 113 113 F F F 25 500-519 64 64 64 112 112 112 F F F 26 520-539 62 62 62 106 106 106 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 65 65 65 113 113 113 F F F 29 580-599 64 64 64 108 108 108 F F F Max gen: 65, Max flush: 113 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.503119805Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.547074506Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.547637132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.547680992Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.547701872Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.547717131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} {"msg":"Crucible 2a1854d6-58c5-441b-8d69-6031e7524c42 has session id: 0776cbc3-efcb-4055-a9ab-6c937367b411","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.548286737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Upstairs opts: Upstairs UUID: 2a1854d6-58c5-441b-8d69-6031e7524c42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.548320977Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Crucible stats registered with UUID: 2a1854d6-58c5-441b-8d69-6031e7524c42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.548351697Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:43.548365527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.548875093Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.548899063Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} {"msg":"2a1854d6-58c5-441b-8d69-6031e7524c42 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.548965862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.548982752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"0","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549009602Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"0","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549024042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"1","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549039202Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"1","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549052241Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549066051Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549083711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549107371Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549131181Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:43.54921627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"ds_connection connected from Ok(127.0.0.1:43542)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.54926147Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:43.54929246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"ds_connection connected from Ok(127.0.0.1:47367)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.54931678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:43.549342029Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549494428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.549950065Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"1","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.550098343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"0","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.550463631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"1","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.55056826Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"0","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.551081986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.551119366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.550644097Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"ds_connection connected from Ok(127.0.0.1:62726)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.550842375Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.551811108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.552348034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[0]R flush_numbers[0..12]: [107, 106, 99, 111, 113, 106, 104, 113, 106, 106, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.552963289Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[0]R generation[0..12]: [63, 62, 58, 64, 65, 62, 61, 65, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.552992139Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553008629Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[1]R flush_numbers[0..12]: [107, 106, 99, 111, 113, 106, 104, 113, 106, 106, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553023139Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[1]R generation[0..12]: [63, 62, 58, 64, 65, 62, 61, 65, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553036979Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553051489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[2]R flush_numbers[0..12]: [107, 106, 99, 111, 113, 106, 104, 113, 106, 106, 102, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553065769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[2]R generation[0..12]: [63, 62, 58, 64, 65, 62, 61, 65, 62, 62, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553079229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553092998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553111178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553124568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553137268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553149978Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"":"downstairs","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553162768Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553174608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"0","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:02:53.553187788Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"1","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553207448Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"client":"2","":"downstairs","session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553220547Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553232597Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"2a1854d6-58c5-441b-8d69-6031e7524c42 is now active with session: 0776cbc3-efcb-4055-a9ab-6c937367b411","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553245157Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553257027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456,"session_id":"0776cbc3-efcb-4055-a9ab-6c937367b411"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553277537Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} {"msg":"Activated sub_volume 2a1854d6-58c5-441b-8d69-6031e7524c42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:53.553289947Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 347 len 32768 data: 4 3 5 5 8 8 9 11 02/30 Read block 238 len 24576 03/30 Write block 108 len 36864 data: 5 5 4 4 4 4 4 4 6 04/30 Read block 473 len 16384 05/30 Write block 444 len 28672 data: 5 4 6 5 5 7 6 06/30 Write block 463 len 32768 data: 8 5 6 5 5 6 5 5 07/30 Write block 319 len 12288 data: 5 6 9 08/30 Read block 497 len 20480 09/30 Read block 296 len 16384 10/30 Write block 11 len 20480 data: 9 8 7 10 11 11/30 Write block 506 len 24576 data: 10 12 10 10 10 9 12/30 Read block 328 len 12288 13/30 Write block 227 len 20480 data: 7 10 8 9 6 14/30 Read block 181 len 4096 15/30 Read block 365 len 28672 16/30 Read block 108 len 32768 17/30 Flush 18/30 Write block 585 len 12288 data: 5 5 6 19/30 Write block 533 len 24576 data: 4 4 3 3 3 3 20/30 Read block 476 len 4096 21/30 Read block 311 len 36864 22/30 Write block 103 len 28672 data: 4 5 5 5 5 6 6 23/30 Write block 290 len 20480 data: 8 8 9 9 7 24/30 Read block 474 len 24576 25/30 Flush 26/30 Write block 8 len 4096 data: 7 27/30 Read block 580 len 28672 28/30 Read block 343 len 4096 29/30 Write block 27 len 28672 data: 8 6 3 4 6 7 8 30/30 Write block 101 len 24576 data: 2 2 5 6 6 6 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:53.574142571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 T T T 1 020-039 62 62 62 106 106 106 T T T 2 040-059 58 58 58 99 99 99 F F F 3 060-079 64 64 64 111 111 111 F F F 4 080-099 65 65 65 113 113 113 F F F 5 100-119 66 66 66 115 115 115 T T T 6 120-139 61 61 61 104 104 104 F F F 7 140-159 65 65 65 113 113 113 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 66 66 66 114 114 114 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 66 66 66 115 115 115 F F F 15 300-319 66 66 66 114 114 114 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 66 66 66 114 114 114 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 66 66 66 114 114 114 F F F 23 460-479 66 66 66 114 114 114 F F F 24 480-499 65 65 65 113 113 113 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 66 66 66 115 115 115 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 65 65 65 113 113 113 F F F 29 580-599 66 66 66 115 115 115 F F F Max gen: 66, Max flush: 115 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 116 116 116 F F F 1 020-039 67 67 67 116 116 116 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 64 64 64 111 111 111 F F F 4 080-099 65 65 65 113 113 113 F F F 5 100-119 67 67 67 116 116 116 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 65 65 65 113 113 113 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 66 66 66 114 114 114 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 66 66 66 115 115 115 F F F 15 300-319 66 66 66 114 114 114 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 66 66 66 114 114 114 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 66 66 66 114 114 114 F F F 23 460-479 66 66 66 114 114 114 F F F 24 480-499 65 65 65 113 113 113 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 66 66 66 115 115 115 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 65 65 65 113 113 113 F F F 29 580-599 66 66 66 115 115 115 F F F Max gen: 67, Max flush: 116 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.801511665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.846620727Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.847226873Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.847268712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.847284732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.847299432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} {"msg":"Crucible 6be4273b-6cc8-4878-a951-9079303f0f7f has session id: 283c9085-2471-48d1-a30b-ceabd51e6902","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.847796258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Upstairs opts: Upstairs UUID: 6be4273b-6cc8-4878-a951-9079303f0f7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.847835258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Crucible stats registered with UUID: 6be4273b-6cc8-4878-a951-9079303f0f7f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.847863788Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:57.847885838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848400324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848425984Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} {"msg":"6be4273b-6cc8-4878-a951-9079303f0f7f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848494903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848513493Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"0","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848540423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"0","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848554493Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"1","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848568513Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"1","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848586163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"2","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848611542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"2","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848635312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848659682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848684932Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"ds_connection connected from Ok(127.0.0.1:53365)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848710782Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"ds_connection connected from Ok(127.0.0.1:42643)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848741631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"ds_connection connected from Ok(127.0.0.1:64087)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.848768121Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.849415526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"0","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.849458096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"2","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.849567705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"1","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850119001Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"0","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850159561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"2","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.85021646Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"1","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850814216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850841876Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[0]R flush_numbers[0..12]: [116, 116, 99, 111, 113, 116, 104, 113, 106, 106, 102, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850857775Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[0]R generation[0..12]: [67, 67, 58, 64, 65, 67, 61, 65, 62, 62, 60, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850872846Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850888045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[1]R flush_numbers[0..12]: [116, 116, 99, 111, 113, 116, 104, 113, 106, 106, 102, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850902025Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[1]R generation[0..12]: [67, 67, 58, 64, 65, 67, 61, 65, 62, 62, 60, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850918905Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850933775Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[2]R flush_numbers[0..12]: [116, 116, 99, 111, 113, 116, 104, 113, 106, 106, 102, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850947835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[2]R generation[0..12]: [67, 67, 58, 64, 65, 67, 61, 65, 62, 62, 60, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850961375Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850975125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.850988654Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851035264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851049404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851062134Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"":"downstairs","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851075374Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851087704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"0","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851100834Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"1","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:02:57.851113904Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"client":"2","":"downstairs","session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851129844Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851142213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"6be4273b-6cc8-4878-a951-9079303f0f7f is now active with session: 283c9085-2471-48d1-a30b-ceabd51e6902","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851157193Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851169383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464,"session_id":"283c9085-2471-48d1-a30b-ceabd51e6902"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851181443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} {"msg":"Activated sub_volume 6be4273b-6cc8-4878-a951-9079303f0f7f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.851192753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 225 len 20480 data: 6 6 8 11 9 02/30 Write block 479 len 8192 data: 4 4 03/30 Read block 308 len 16384 04/30 Read block 291 len 8192 05/30 Write block 405 len 8192 data: 10 10 06/30 Flush 07/30 Write block 289 len 20480 data: 8 9 9 10 10 08/30 Read block 508 len 4096 09/30 Read block 305 len 16384 10/30 Read block 351 len 4096 11/30 Write block 75 len 36864 data: 8 8 6 7 7 5 4 3 2 12/30 Flush 13/30 Write block 70 len 8192 data: 4 5 14/30 Read block 419 len 28672 15/30 Read block 369 len 40960 16/30 Write block 347 len 32768 data: 5 4 6 6 9 9 10 12 17/30 Read block 96 len 40960 18/30 Write block 448 len 36864 data: 6 8 7 6 5 5 7 7 6 19/30 Write block 81 len 20480 data: 5 4 3 4 4 20/30 Read block 187 len 36864 21/30 Read block 555 len 24576 22/30 Write block 65 len 24576 data: 4 4 4 3 4 5 23/30 Write block 300 len 36864 data: 7 7 6 5 5 5 4 5 5 24/30 Read block 0 len 24576 25/30 Read block 420 len 20480 26/30 Read block 362 len 16384 27/30 Write block 90 len 28672 data: 7 6 8 8 7 7 6 28/30 Write block 404 len 8192 data: 12 11 29/30 Read block 61 len 4096 30/30 Read block 85 len 16384 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 2 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 18 17 18 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:57.871758039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 116 116 116 F F F 1 020-039 67 67 67 116 116 116 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 68 68 68 118 118 118 T T T 4 080-099 68 68 68 118 118 118 T T T 5 100-119 67 67 67 116 116 116 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 65 65 65 113 113 113 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 68 68 68 117 117 117 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 68 68 68 118 118 118 F F F 15 300-319 66 66 66 114 114 114 T T T 16 320-339 66 66 66 114 114 114 F F F 17 340-359 66 66 66 114 114 114 T T T 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 68 68 68 117 117 117 T T T 21 420-439 56 56 56 98 98 98 F F F 22 440-459 66 66 66 114 114 114 T T T 23 460-479 68 68 68 117 117 117 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 66 66 66 115 115 115 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 65 65 65 113 113 113 F F F 29 580-599 66 66 66 115 115 115 F F F Max gen: 68, Max flush: 118 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 116 116 116 F F F 1 020-039 67 67 67 116 116 116 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 69 69 69 119 119 119 F F F 4 080-099 69 69 69 119 119 119 F F F 5 100-119 67 67 67 116 116 116 F F F 6 120-139 61 61 61 104 104 104 F F F 7 140-159 65 65 65 113 113 113 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 68 68 68 117 117 117 F F F 12 240-259 47 47 47 80 80 80 F F F 13 260-279 61 61 61 104 104 104 F F F 14 280-299 68 68 68 118 118 118 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 69 69 69 119 119 119 F F F 18 360-379 60 60 60 102 102 102 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 69 69 69 119 119 119 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 69 69 69 119 119 119 F F F 23 460-479 68 68 68 117 117 117 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 66 66 66 115 115 115 F F F 27 540-559 65 65 65 113 113 113 F F F 28 560-579 65 65 65 113 113 113 F F F 29 580-599 66 66 66 115 115 115 F F F Max gen: 69, Max flush: 119 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.108860522Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.153647086Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.154224592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.154264571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.154279062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.154293241Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} {"msg":"Crucible cb8cc6fa-20f9-47a3-8232-104aefb0e10d has session id: d140c584-7548-4683-a8f9-1d4199e1c0d1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.154818687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Upstairs opts: Upstairs UUID: cb8cc6fa-20f9-47a3-8232-104aefb0e10d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.154871377Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Crucible stats registered with UUID: cb8cc6fa-20f9-47a3-8232-104aefb0e10d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.154899437Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:02.154922647Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155412143Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155437783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} {"msg":"cb8cc6fa-20f9-47a3-8232-104aefb0e10d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155503212Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155521832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"0","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155548532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"0","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155566582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"1","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155584372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"1","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155609361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"2","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155637051Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"2","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155657761Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155672261Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155695431Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57783)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.15575818Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39882)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.155988559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36557)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.156021348Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.156494675Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"0","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.156534225Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"1","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.156565004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"2","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.15717082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"1","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.1572151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"0","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157244499Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"2","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157796925Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157823235Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[0]R flush_numbers[0..12]: [116, 116, 99, 119, 119, 116, 104, 113, 106, 106, 102, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157865784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[0]R generation[0..12]: [67, 67, 58, 69, 69, 67, 61, 65, 62, 62, 60, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157884784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157899164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[1]R flush_numbers[0..12]: [116, 116, 99, 119, 119, 116, 104, 113, 106, 106, 102, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157912924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[1]R generation[0..12]: [67, 67, 58, 69, 69, 67, 61, 65, 62, 62, 60, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157926314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157940404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[2]R flush_numbers[0..12]: [116, 116, 99, 119, 119, 116, 104, 113, 106, 106, 102, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157954054Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[2]R generation[0..12]: [67, 67, 58, 69, 69, 67, 61, 65, 62, 62, 60, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157967384Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157981254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.157994144Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158011603Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158035273Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158059173Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"":"downstairs","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158076813Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158089183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"0","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158102543Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"1","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158118183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"client":"2","":"downstairs","session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158142422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158158042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"cb8cc6fa-20f9-47a3-8232-104aefb0e10d is now active with session: d140c584-7548-4683-a8f9-1d4199e1c0d1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158170972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158183052Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472,"session_id":"d140c584-7548-4683-a8f9-1d4199e1c0d1"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:02.158202252Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} {"msg":"Activated sub_volume cb8cc6fa-20f9-47a3-8232-104aefb0e10d","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-18T05:03:02.158222802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 12288 data: 5 4 4 02/30 Write block 470 len 4096 data: 6 03/30 Write block 65 len 20480 data: 5 5 5 4 5 04/30 Write block 531 len 32768 data: 4 4 5 5 4 4 4 4 05/30 Read block 59 len 36864 06/30 Flush 07/30 Write block 113 len 32768 data: 5 5 5 7 8 9 8 7 08/30 Read block 354 len 20480 09/30 Write block 152 len 8192 data: 6 5 10/30 Flush 11/30 Write block 288 len 32768 data: 9 9 10 10 11 11 8 4 12/30 Read block 129 len 16384 13/30 Write block 253 len 8192 data: 4 6 14/30 Read block 50 len 24576 15/30 Read block 424 len 24576 16/30 Write block 88 len 28672 data: 5 6 8 7 9 9 8 17/30 Read block 89 len 40960 18/30 Write block 213 len 32768 data: 5 6 6 7 7 5 5 3 19/30 Write block 214 len 28672 data: 7 7 8 8 6 6 4 20/30 Flush 21/30 Write block 223 len 4096 data: 5 22/30 Flush 23/30 Write block 126 len 32768 data: 6 6 4 4 5 8 8 7 24/30 Write block 531 len 4096 data: 5 25/30 Read block 140 len 24576 26/30 Write block 593 len 24576 data: 6 5 6 5 5 5 27/30 Write block 275 len 8192 data: 6 4 28/30 Write block 291 len 12288 data: 11 12 12 29/30 Write block 359 len 8192 data: 7 5 30/30 Write block 545 len 16384 data: 8 9 11 8 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:02.17590204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 116 116 116 F F F 1 020-039 67 67 67 116 116 116 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 70 70 70 122 122 122 F F F 5 100-119 70 70 70 121 121 121 F F F 6 120-139 70 70 70 121 121 121 T T T 7 140-159 70 70 70 121 121 121 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 70 70 70 122 122 122 F F F 13 260-279 61 61 61 104 104 104 T T T 14 280-299 70 70 70 122 122 122 T T T 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 69 69 69 119 119 119 T T T 18 360-379 60 60 60 102 102 102 T T T 19 380-399 62 62 62 106 106 106 F F F 20 400-419 69 69 69 119 119 119 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 69 69 69 119 119 119 F F F 23 460-479 70 70 70 120 120 120 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 70 70 70 120 120 120 T T T 27 540-559 65 65 65 113 113 113 T T T 28 560-579 65 65 65 113 113 113 F F F 29 580-599 66 66 66 115 115 115 T T T Max gen: 70, Max flush: 123 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 116 116 116 F F F 1 020-039 67 67 67 116 116 116 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 70 70 70 122 122 122 F F F 5 100-119 70 70 70 121 121 121 F F F 6 120-139 71 71 71 124 124 124 F F F 7 140-159 70 70 70 121 121 121 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 70 70 70 122 122 122 F F F 13 260-279 71 71 71 124 124 124 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 71 71 71 124 124 124 F F F 18 360-379 71 71 71 124 124 124 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 69 69 69 119 119 119 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 69 69 69 119 119 119 F F F 23 460-479 70 70 70 120 120 120 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 71 71 71 124 124 124 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 65 65 65 113 113 113 F F F 29 580-599 71 71 71 124 124 124 F F F Max gen: 71, Max flush: 124 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.432311924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.475867638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.476482453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.476526163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.476541382Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.476555693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} {"msg":"Crucible e704a079-f5a3-4c9d-b059-b60dea160fcf has session id: ef7706ce-0fe8-4b80-a34d-f44e9b7c591a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477059189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Upstairs opts: Upstairs UUID: e704a079-f5a3-4c9d-b059-b60dea160fcf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477098108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Crucible stats registered with UUID: e704a079-f5a3-4c9d-b059-b60dea160fcf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477124358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:16.477138688Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477608674Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477632884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} {"msg":"e704a079-f5a3-4c9d-b059-b60dea160fcf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477715964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477734214Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"0","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477761213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"0","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477775083Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"1","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477789243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"1","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477801923Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"2","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477821913Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"2","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477848153Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477874172Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477899052Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46876)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.477984822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40823)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.478023411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63656)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.478052681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.478750766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"0","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.478780576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"1","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.479285762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"2","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.479341261Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"0","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.479412061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"1","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.479972467Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480083406Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"2","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480111246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[0]R flush_numbers[0..12]: [116, 116, 99, 120, 122, 121, 124, 121, 106, 106, 122, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480653242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[0]R generation[0..12]: [67, 67, 58, 70, 70, 70, 71, 70, 62, 62, 70, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480680142Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480695831Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[1]R flush_numbers[0..12]: [116, 116, 99, 120, 122, 121, 124, 121, 106, 106, 122, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480709881Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[1]R generation[0..12]: [67, 67, 58, 70, 70, 70, 71, 70, 62, 62, 70, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480726721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480741081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[2]R flush_numbers[0..12]: [116, 116, 99, 120, 122, 121, 124, 121, 106, 106, 122, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480754911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[2]R generation[0..12]: [67, 67, 58, 70, 70, 70, 71, 70, 62, 62, 70, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480768291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480782231Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480794951Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480807801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Next flush: 125","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-03-18T05:03:16.48082737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.4808478Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"":"downstairs","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.4808614Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.48087338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"0","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.48088666Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"1","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.48089956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"client":"2","":"downstairs","session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.48091257Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.48092467Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"e704a079-f5a3-4c9d-b059-b60dea160fcf is now active with session: ef7706ce-0fe8-4b80-a34d-f44e9b7c591a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.48093734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480952669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480,"session_id":"ef7706ce-0fe8-4b80-a34d-f44e9b7c591a"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480965269Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} {"msg":"Activated sub_volume e704a079-f5a3-4c9d-b059-b60dea160fcf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.480976389Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 28672 data: 6 4 4 4 2 2 2 02/30 Read block 47 len 36864 03/30 Read block 295 len 8192 04/30 Read block 247 len 8192 05/30 Flush 06/30 Write block 245 len 16384 data: 3 3 4 6 07/30 Flush 08/30 Write block 492 len 28672 data: 9 8 8 8 5 7 8 09/30 Write block 14 len 32768 data: 11 12 8 8 6 8 6 7 10/30 Write block 254 len 32768 data: 7 6 4 4 4 4 4 4 11/30 Write block 1 len 28672 data: 5 5 6 5 5 5 7 12/30 Read block 129 len 8192 13/30 Read block 165 len 8192 14/30 Write block 595 len 12288 data: 7 6 6 15/30 Write block 129 len 32768 data: 5 6 9 9 8 7 6 6 16/30 Read block 391 len 20480 17/30 Read block 77 len 4096 18/30 Write block 375 len 16384 data: 6 5 4 7 19/30 Read block 19 len 20480 20/30 Read block 18 len 28672 21/30 Write block 183 len 20480 data: 7 7 7 5 5 22/30 Read block 211 len 40960 23/30 Write block 575 len 24576 data: 5 6 6 5 6 6 24/30 Read block 166 len 8192 25/30 Read block 520 len 12288 26/30 Write block 166 len 36864 data: 10 10 7 8 8 6 6 7 6 27/30 Write block 156 len 8192 data: 4 5 28/30 Read block 308 len 20480 29/30 Write block 195 len 32768 data: 6 6 6 4 5 5 6 5 30/30 Read block 230 len 40960 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 7 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 23 22 22 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:16.501169758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 116 116 116 T T T 1 020-039 67 67 67 116 116 116 T T T 2 040-059 58 58 58 99 99 99 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 70 70 70 122 122 122 F F F 5 100-119 70 70 70 121 121 121 F F F 6 120-139 71 71 71 124 124 124 T T T 7 140-159 72 72 72 125 125 125 T T T 8 160-179 62 62 62 106 106 106 T T T 9 180-199 62 62 62 106 106 106 T T T 10 200-219 70 70 70 122 122 122 T T T 11 220-239 70 70 70 123 123 123 F F F 12 240-259 72 72 72 126 126 126 T T T 13 260-279 71 71 71 124 124 124 T T T 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 71 71 71 124 124 124 F F F 18 360-379 71 71 71 124 124 124 T T T 19 380-399 62 62 62 106 106 106 F F F 20 400-419 69 69 69 119 119 119 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 69 69 69 119 119 119 F F F 23 460-479 70 70 70 120 120 120 F F F 24 480-499 68 68 68 117 117 117 T T T 25 500-519 66 66 66 114 114 114 F F F 26 520-539 71 71 71 124 124 124 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 65 65 65 113 113 113 T T T 29 580-599 71 71 71 124 124 124 T T T Max gen: 72, Max flush: 126 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 127 127 127 F F F 1 020-039 73 73 73 127 127 127 F F F 2 040-059 58 58 58 99 99 99 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 70 70 70 122 122 122 F F F 5 100-119 70 70 70 121 121 121 F F F 6 120-139 73 73 73 127 127 127 F F F 7 140-159 73 73 73 127 127 127 F F F 8 160-179 73 73 73 127 127 127 F F F 9 180-199 73 73 73 127 127 127 F F F 10 200-219 73 73 73 127 127 127 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 73 73 73 127 127 127 F F F 13 260-279 73 73 73 127 127 127 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 71 71 71 124 124 124 F F F 18 360-379 73 73 73 127 127 127 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 69 69 69 119 119 119 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 69 69 69 119 119 119 F F F 23 460-479 70 70 70 120 120 120 F F F 24 480-499 73 73 73 127 127 127 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 71 71 71 124 124 124 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 73 73 73 127 127 127 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 73, Max flush: 127 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.779593401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.824264497Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.824896872Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.824942161Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.824957971Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.824972671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} {"msg":"Crucible c2e766fa-a30e-47f3-ba55-e9a820c1d855 has session id: aeb21e6e-a8a0-4f0a-a7f2-7202a4908905","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.825553667Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Upstairs opts: Upstairs UUID: c2e766fa-a30e-47f3-ba55-e9a820c1d855, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.825584317Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Crucible stats registered with UUID: c2e766fa-a30e-47f3-ba55-e9a820c1d855","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.825600616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:20.825613496Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826159252Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826182802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} {"msg":"c2e766fa-a30e-47f3-ba55-e9a820c1d855 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826256742Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826275031Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"0","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826307881Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"0","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826325631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"1","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826352551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"1","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.82641297Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"2","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.82645509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"2","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.82648315Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.82651104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.82653536Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"ds_connection connected from Ok(127.0.0.1:61366)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826559339Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"ds_connection connected from Ok(127.0.0.1:62624)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826585539Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"ds_connection connected from Ok(127.0.0.1:60534)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.826633489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.827244184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"0","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.827272834Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"1","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.827954199Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"1","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.827982249Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"0","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.828610794Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.828716473Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.82917283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"2","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.829688506Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"2","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[0]R flush_numbers[0..12]: [127, 127, 99, 120, 122, 121, 127, 127, 127, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830204032Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[0]R generation[0..12]: [73, 73, 58, 70, 70, 70, 73, 73, 73, 73, 73, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830229292Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830246812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[1]R flush_numbers[0..12]: [127, 127, 99, 120, 122, 121, 127, 127, 127, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830269821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[1]R generation[0..12]: [73, 73, 58, 70, 70, 70, 73, 73, 73, 73, 73, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830295631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830322851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[2]R flush_numbers[0..12]: [127, 127, 99, 120, 122, 121, 127, 127, 127, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830342591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[2]R generation[0..12]: [73, 73, 58, 70, 70, 70, 73, 73, 73, 73, 73, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830367681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830395031Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.83042312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.83043952Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.83045651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"All extents match","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-18T05:03:20.83048589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"":"downstairs","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.8305119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.83053283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"0","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.83054769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"1","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830561909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"client":"2","":"downstairs","session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830585359Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830608679Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"c2e766fa-a30e-47f3-ba55-e9a820c1d855 is now active with session: aeb21e6e-a8a0-4f0a-a7f2-7202a4908905","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830633339Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830648669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488,"session_id":"aeb21e6e-a8a0-4f0a-a7f2-7202a4908905"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830661659Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} {"msg":"Activated sub_volume c2e766fa-a30e-47f3-ba55-e9a820c1d855","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:20.830680738Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 442 len 4096 data: 4 02/30 Read block 75 len 16384 03/30 Write block 22 len 32768 data: 7 10 9 9 10 9 7 4 04/30 Write block 148 len 20480 data: 3 4 5 6 7 05/30 Read block 442 len 8192 06/30 Read block 329 len 20480 07/30 Read block 10 len 36864 08/30 Flush 09/30 Write block 99 len 12288 data: 5 5 3 10/30 Write block 343 len 4096 data: 4 11/30 Read block 1 len 24576 12/30 Read block 523 len 28672 13/30 Write block 136 len 20480 data: 7 6 5 4 7 14/30 Flush 15/30 Write block 84 len 36864 data: 5 5 5 6 6 7 9 8 10 16/30 Read block 79 len 20480 17/30 Read block 226 len 24576 18/30 Read block 113 len 16384 19/30 Write block 5 len 28672 data: 6 6 8 8 9 8 10 20/30 Read block 80 len 20480 21/30 Read block 480 len 24576 22/30 Read block 573 len 40960 23/30 Read block 175 len 4096 24/30 Flush 25/30 Write block 37 len 28672 data: 9 10 10 7 6 4 5 26/30 Write block 179 len 8192 data: 8 7 27/30 Read block 449 len 4096 28/30 Read block 503 len 20480 29/30 Write block 455 len 16384 data: 8 7 7 3 30/30 Write block 185 len 8192 data: 8 6 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:20.850358681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 F F F 1 020-039 74 74 74 128 128 128 T T T 2 040-059 58 58 58 99 99 99 T T T 3 060-079 70 70 70 120 120 120 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 74 74 74 129 129 129 F F F 6 120-139 74 74 74 129 129 129 F F F 7 140-159 74 74 74 129 129 129 F F F 8 160-179 73 73 73 127 127 127 T T T 9 180-199 73 73 73 127 127 127 T T T 10 200-219 73 73 73 127 127 127 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 73 73 73 127 127 127 F F F 13 260-279 73 73 73 127 127 127 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 74 74 74 129 129 129 F F F 18 360-379 73 73 73 127 127 127 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 69 69 69 119 119 119 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 74 74 74 128 128 128 T T T 23 460-479 70 70 70 120 120 120 F F F 24 480-499 73 73 73 127 127 127 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 71 71 71 124 124 124 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 73 73 73 127 127 127 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 74, Max flush: 130 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 F F F 1 020-039 75 75 75 131 131 131 F F F 2 040-059 75 75 75 131 131 131 F F F 3 060-079 70 70 70 120 120 120 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 74 74 74 129 129 129 F F F 6 120-139 74 74 74 129 129 129 F F F 7 140-159 74 74 74 129 129 129 F F F 8 160-179 75 75 75 131 131 131 F F F 9 180-199 75 75 75 131 131 131 F F F 10 200-219 73 73 73 127 127 127 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 73 73 73 127 127 127 F F F 13 260-279 73 73 73 127 127 127 F F F 14 280-299 71 71 71 124 124 124 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 74 74 74 129 129 129 F F F 18 360-379 73 73 73 127 127 127 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 69 69 69 119 119 119 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 75 75 75 131 131 131 F F F 23 460-479 70 70 70 120 120 120 F F F 24 480-499 73 73 73 127 127 127 F F F 25 500-519 66 66 66 114 114 114 F F F 26 520-539 71 71 71 124 124 124 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 73 73 73 127 127 127 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 75, Max flush: 131 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.085964993Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:25.129420458Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.129781845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.161749096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.162283112Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.162321731Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.162338121Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.162355461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} {"msg":"Crucible bc4da4fb-3b51-40e0-9720-38c682674088 has session id: 6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.162879797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Upstairs opts: Upstairs UUID: bc4da4fb-3b51-40e0-9720-38c682674088, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.162921577Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Crucible stats registered with UUID: bc4da4fb-3b51-40e0-9720-38c682674088","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.162946757Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:25.162967676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163370093Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163393943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} {"msg":"bc4da4fb-3b51-40e0-9720-38c682674088 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163466003Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163484123Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"0","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163511352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"0","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163524972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"1","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163539362Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"1","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163552762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"2","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163566752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"2","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163579512Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163592782Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163616332Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58644)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163647442Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51676)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.163676351Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61350)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.16380171Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.164403066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"2","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.164444276Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"1","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.164574095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"0","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165075791Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"1","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"2","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.16522826Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"0","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165650027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165756306Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[0]R flush_numbers[0..12]: [130, 131, 131, 120, 130, 129, 129, 129, 131, 131, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165840625Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[0]R generation[0..12]: [74, 75, 75, 70, 74, 74, 74, 74, 75, 75, 73, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165861785Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165876665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[1]R flush_numbers[0..12]: [130, 131, 131, 120, 130, 129, 129, 129, 131, 131, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165895025Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[1]R generation[0..12]: [74, 75, 75, 70, 74, 74, 74, 74, 75, 75, 73, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165909094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165923054Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[2]R flush_numbers[0..12]: [130, 131, 131, 120, 130, 129, 129, 129, 131, 131, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165936604Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[2]R generation[0..12]: [74, 75, 75, 70, 74, 74, 74, 74, 75, 75, 73, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165949984Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165963744Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165976444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.165989184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166001794Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166014534Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"":"downstairs","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:25.166027334Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166044404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"0","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166057433Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"1","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166070353Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"client":"2","":"downstairs","session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166083273Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166097363Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"bc4da4fb-3b51-40e0-9720-38c682674088 is now active with session: 6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166110233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166122223Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496,"session_id":"6cc64fcf-bc5e-4d97-b44d-94a154aa3e5c"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166134193Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} {"msg":"Activated sub_volume bc4da4fb-3b51-40e0-9720-38c682674088","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:25.166145283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 287 len 8192 data: 9 10 02/30 Read block 88 len 28672 03/30 Read block 168 len 32768 04/30 Write block 192 len 4096 data: 6 05/30 Write block 57 len 16384 data: 7 7 6 5 06/30 Flush 07/30 Write block 423 len 36864 data: 7 9 7 5 5 4 6 7 11 08/30 Write block 207 len 32768 data: 5 6 5 6 6 5 6 8 09/30 Write block 195 len 8192 data: 7 7 10/30 Read block 305 len 36864 11/30 Read block 452 len 36864 12/30 Flush 13/30 Write block 415 len 8192 data: 5 5 14/30 Read block 414 len 12288 15/30 Read block 500 len 28672 16/30 Write block 344 len 28672 data: 4 4 3 6 5 7 7 17/30 Read block 381 len 8192 18/30 Write block 512 len 32768 data: 6 5 2 3 3 5 7 9 19/30 Write block 351 len 40960 data: 10 10 11 13 9 9 8 7 8 6 20/30 Read block 488 len 32768 21/30 Read block 318 len 40960 22/30 Write block 153 len 16384 data: 6 6 4 5 23/30 Flush 24/30 Write block 210 len 24576 data: 7 7 6 7 9 8 25/30 Read block 535 len 36864 26/30 Read block 557 len 16384 27/30 Write block 261 len 28672 data: 5 5 7 6 8 9 9 28/30 Read block 587 len 4096 29/30 Read block 346 len 32768 30/30 Read block 32 len 16384 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:25.186796728Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 F F F 1 020-039 75 75 75 131 131 131 F F F 2 040-059 76 76 76 132 132 132 F F F 3 060-079 76 76 76 132 132 132 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 74 74 74 129 129 129 F F F 6 120-139 74 74 74 129 129 129 F F F 7 140-159 76 76 76 134 134 134 F F F 8 160-179 75 75 75 131 131 131 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 76 76 76 133 133 133 T T T 11 220-239 70 70 70 123 123 123 F F F 12 240-259 73 73 73 127 127 127 F F F 13 260-279 73 73 73 127 127 127 T T T 14 280-299 76 76 76 132 132 132 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 76 76 76 134 134 134 F F F 18 360-379 76 76 76 134 134 134 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 76 76 76 134 134 134 F F F 21 420-439 76 76 76 133 133 133 F F F 22 440-459 75 75 75 131 131 131 F F F 23 460-479 70 70 70 120 120 120 F F F 24 480-499 73 73 73 127 127 127 F F F 25 500-519 76 76 76 134 134 134 F F F 26 520-539 71 71 71 124 124 124 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 73 73 73 127 127 127 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 76, Max flush: 134 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 F F F 1 020-039 75 75 75 131 131 131 F F F 2 040-059 76 76 76 132 132 132 F F F 3 060-079 76 76 76 132 132 132 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 74 74 74 129 129 129 F F F 6 120-139 74 74 74 129 129 129 F F F 7 140-159 76 76 76 134 134 134 F F F 8 160-179 75 75 75 131 131 131 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 77 77 77 135 135 135 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 73 73 73 127 127 127 F F F 13 260-279 77 77 77 135 135 135 F F F 14 280-299 76 76 76 132 132 132 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 76 76 76 134 134 134 F F F 18 360-379 76 76 76 134 134 134 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 76 76 76 134 134 134 F F F 21 420-439 76 76 76 133 133 133 F F F 22 440-459 75 75 75 131 131 131 F F F 23 460-479 70 70 70 120 120 120 F F F 24 480-499 73 73 73 127 127 127 F F F 25 500-519 76 76 76 134 134 134 F F F 26 520-539 71 71 71 124 124 124 F F F 27 540-559 71 71 71 124 124 124 F F F 28 560-579 73 73 73 127 127 127 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 77, Max flush: 135 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.435498544Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.479183407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.479751752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.479796002Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.479811312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.479826012Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} {"msg":"Crucible 0a5358c1-c8c6-4af1-a42d-fab2b7715168 has session id: 2c7547e9-8385-44b9-87cf-c901806626af","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.480326158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Upstairs opts: Upstairs UUID: 0a5358c1-c8c6-4af1-a42d-fab2b7715168, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.480365628Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Crucible stats registered with UUID: 0a5358c1-c8c6-4af1-a42d-fab2b7715168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.480389987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:29.480411527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.480870944Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.480905474Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} {"msg":"0a5358c1-c8c6-4af1-a42d-fab2b7715168 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.480991313Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481027523Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"0","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481070582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"0","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481143212Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"1","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481169762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"1","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481196872Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"2","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481214781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"2","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481229041Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481245861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481270801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"ds_connection connected from Ok(127.0.0.1:46534)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481297511Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"ds_connection connected from Ok(127.0.0.1:37388)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48132156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"ds_connection connected from Ok(127.0.0.1:36018)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48139406Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.481976436Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"0","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.482018125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"1","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.482629561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"0","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.482658561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"2","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48267455Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"1","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483294616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483323705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483339145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"2","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[0]R flush_numbers[0..12]: [130, 131, 132, 132, 130, 129, 129, 134, 131, 133, 135, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483850992Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[0]R generation[0..12]: [74, 75, 76, 76, 74, 74, 74, 76, 75, 76, 77, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483880161Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483896761Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[1]R flush_numbers[0..12]: [130, 131, 132, 132, 130, 129, 129, 134, 131, 133, 135, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483911691Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[1]R generation[0..12]: [74, 75, 76, 76, 74, 74, 74, 76, 75, 76, 77, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483925781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483939931Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[2]R flush_numbers[0..12]: [130, 131, 132, 132, 130, 129, 129, 134, 131, 133, 135, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483954131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[2]R generation[0..12]: [74, 75, 76, 76, 74, 74, 74, 76, 75, 76, 77, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483968041Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.483981981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48399472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48400788Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48402748Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48404495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"":"downstairs","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48405863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48407053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"0","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48408352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"1","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48409644Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"client":"2","":"downstairs","session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48410949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48412181Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"0a5358c1-c8c6-4af1-a42d-fab2b7715168 is now active with session: 2c7547e9-8385-44b9-87cf-c901806626af","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.48413461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.484146509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504,"session_id":"2c7547e9-8385-44b9-87cf-c901806626af"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:03:29.484158589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} {"msg":"Activated sub_volume 0a5358c1-c8c6-4af1-a42d-fab2b7715168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:29.484172019Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 288 len 24576 data: 11 10 11 12 13 13 02/30 Write block 569 len 12288 data: 7 8 6 03/30 Read block 212 len 12288 04/30 Write block 386 len 32768 data: 6 4 4 4 3 3 3 2 05/30 Read block 300 len 24576 06/30 Read block 162 len 4096 07/30 Read block 225 len 40960 08/30 Write block 552 len 24576 data: 7 7 6 8 9 11 09/30 Read block 355 len 40960 10/30 Read block 261 len 28672 11/30 Read block 482 len 4096 12/30 Read block 548 len 12288 13/30 Write block 409 len 24576 data: 9 8 8 7 7 5 14/30 Read block 202 len 36864 15/30 Read block 213 len 28672 16/30 Read block 176 len 40960 17/30 Write block 130 len 20480 data: 7 10 10 9 8 18/30 Read block 397 len 20480 19/30 Read block 511 len 24576 20/30 Read block 74 len 8192 21/30 Write block 364 len 24576 data: 4 4 5 6 6 5 22/30 Read block 465 len 8192 23/30 Write block 433 len 8192 data: 9 9 24/30 Write block 371 len 8192 data: 6 7 25/30 Read block 310 len 24576 26/30 Write block 483 len 8192 data: 9 9 27/30 Read block 137 len 4096 28/30 Write block 523 len 28672 data: 10 9 8 9 7 5 5 29/30 Write block 463 len 12288 data: 9 6 7 30/30 Write block 381 len 28672 data: 6 5 4 5 6 7 5 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:29.503600844Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 F F F 1 020-039 75 75 75 131 131 131 F F F 2 040-059 76 76 76 132 132 132 F F F 3 060-079 76 76 76 132 132 132 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 74 74 74 129 129 129 F F F 6 120-139 74 74 74 129 129 129 T T T 7 140-159 76 76 76 134 134 134 F F F 8 160-179 75 75 75 131 131 131 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 77 77 77 135 135 135 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 73 73 73 127 127 127 F F F 13 260-279 77 77 77 135 135 135 F F F 14 280-299 76 76 76 132 132 132 T T T 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 76 76 76 134 134 134 F F F 18 360-379 76 76 76 134 134 134 T T T 19 380-399 62 62 62 106 106 106 T T T 20 400-419 76 76 76 134 134 134 T T T 21 420-439 76 76 76 133 133 133 T T T 22 440-459 75 75 75 131 131 131 F F F 23 460-479 70 70 70 120 120 120 T T T 24 480-499 73 73 73 127 127 127 T T T 25 500-519 76 76 76 134 134 134 F F F 26 520-539 71 71 71 124 124 124 T T T 27 540-559 71 71 71 124 124 124 T T T 28 560-579 73 73 73 127 127 127 T T T 29 580-599 73 73 73 127 127 127 F F F Max gen: 77, Max flush: 135 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 F F F 1 020-039 75 75 75 131 131 131 F F F 2 040-059 76 76 76 132 132 132 F F F 3 060-079 76 76 76 132 132 132 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 74 74 74 129 129 129 F F F 6 120-139 78 78 78 136 136 136 F F F 7 140-159 76 76 76 134 134 134 F F F 8 160-179 75 75 75 131 131 131 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 77 77 77 135 135 135 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 73 73 73 127 127 127 F F F 13 260-279 77 77 77 135 135 135 F F F 14 280-299 78 78 78 136 136 136 F F F 15 300-319 69 69 69 119 119 119 F F F 16 320-339 66 66 66 114 114 114 F F F 17 340-359 76 76 76 134 134 134 F F F 18 360-379 78 78 78 136 136 136 F F F 19 380-399 78 78 78 136 136 136 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 78 78 78 136 136 136 F F F 22 440-459 75 75 75 131 131 131 F F F 23 460-479 78 78 78 136 136 136 F F F 24 480-499 78 78 78 136 136 136 F F F 25 500-519 76 76 76 134 134 134 F F F 26 520-539 78 78 78 136 136 136 F F F 27 540-559 78 78 78 136 136 136 F F F 28 560-579 78 78 78 136 136 136 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 78, Max flush: 136 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.76790922Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.812879843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.813600907Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.813655437Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.813673197Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.813691187Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} {"msg":"Crucible 8fc18a0d-716e-4c14-8fd1-0c0b4788588f has session id: ee8c8c07-6d47-4520-903f-911db2520e94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814290502Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Upstairs opts: Upstairs UUID: 8fc18a0d-716e-4c14-8fd1-0c0b4788588f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814320422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Crucible stats registered with UUID: 8fc18a0d-716e-4c14-8fd1-0c0b4788588f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814335822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:43.814348522Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814790619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814824578Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} {"msg":"8fc18a0d-716e-4c14-8fd1-0c0b4788588f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814882358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814909478Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"0","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814937887Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"0","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814952477Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"1","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814966917Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"1","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.814980207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"2","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815003977Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"2","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815026887Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815050097Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815080076Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"ds_connection connected from Ok(127.0.0.1:40898)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815105636Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"ds_connection connected from Ok(127.0.0.1:46160)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815302045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"ds_connection connected from Ok(127.0.0.1:47156)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815339224Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.815821861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"0","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.816042559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"1","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.816114918Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"2","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.816618845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"0","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.816658185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"2","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.816776373Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"1","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.81723186Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.81732076Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[0]R flush_numbers[0..12]: [130, 131, 132, 132, 130, 129, 136, 134, 131, 133, 135, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817443719Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[0]R generation[0..12]: [74, 75, 76, 76, 74, 74, 78, 76, 75, 76, 77, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817464279Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817483808Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[1]R flush_numbers[0..12]: [130, 131, 132, 132, 130, 129, 136, 134, 131, 133, 135, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817512938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[1]R generation[0..12]: [74, 75, 76, 76, 74, 74, 78, 76, 75, 76, 77, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817526848Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817540618Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[2]R flush_numbers[0..12]: [130, 131, 132, 132, 130, 129, 136, 134, 131, 133, 135, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817554238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[2]R generation[0..12]: [74, 75, 76, 76, 74, 74, 78, 76, 75, 76, 77, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817567738Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817581318Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Max found gen is 79","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817594098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Generation requested: 80 >= found:79","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817607187Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817619627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"All extents match","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2025-03-18T05:03:43.817638957Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"":"downstairs","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817651967Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817663717Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"0","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817676737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"1","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817689617Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"client":"2","":"downstairs","session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817704687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817718006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"8fc18a0d-716e-4c14-8fd1-0c0b4788588f is now active with session: ee8c8c07-6d47-4520-903f-911db2520e94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817730697Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817742596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512,"session_id":"ee8c8c07-6d47-4520-903f-911db2520e94"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817754736Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} {"msg":"Activated sub_volume 8fc18a0d-716e-4c14-8fd1-0c0b4788588f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:43.817765936Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 249 len 12288 data: 6 6 5 02/30 Write block 304 len 16384 data: 6 6 5 6 03/30 Write block 204 len 16384 data: 4 5 5 6 04/30 Write block 572 len 28672 data: 6 6 4 6 7 7 6 05/30 Read block 314 len 24576 06/30 Read block 475 len 12288 07/30 Write block 433 len 28672 data: 10 10 9 7 8 8 8 08/30 Read block 356 len 36864 09/30 Read block 209 len 32768 10/30 Read block 37 len 12288 11/30 Write block 33 len 28672 data: 9 9 9 11 10 11 11 12/30 Read block 22 len 36864 13/30 Read block 492 len 8192 14/30 Read block 169 len 8192 15/30 Flush 16/30 Write block 317 len 16384 data: 6 5 6 7 17/30 Read block 416 len 28672 18/30 Write block 201 len 8192 data: 7 6 19/30 Write block 316 len 24576 data: 7 7 6 7 8 10 20/30 Write block 263 len 12288 data: 8 7 9 21/30 Flush 22/30 Write block 163 len 32768 data: 6 5 8 11 11 8 9 9 23/30 Write block 17 len 20480 data: 9 7 9 7 8 24/30 Write block 511 len 12288 data: 10 7 6 25/30 Read block 444 len 28672 26/30 Write block 438 len 4096 data: 9 27/30 Write block 253 len 24576 data: 5 8 7 5 5 5 28/30 Write block 439 len 32768 data: 9 6 4 5 6 6 5 7 29/30 Write block 119 len 28672 data: 9 8 6 6 7 9 8 30/30 Write block 126 len 40960 data: 7 7 5 6 8 11 11 10 9 7 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:43.836641815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 T T T 1 020-039 80 80 80 137 137 137 T T T 2 040-059 76 76 76 132 132 132 F F F 3 060-079 76 76 76 132 132 132 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 74 74 74 129 129 129 T T T 6 120-139 78 78 78 136 136 136 T T T 7 140-159 76 76 76 134 134 134 F F F 8 160-179 75 75 75 131 131 131 T T T 9 180-199 76 76 76 133 133 133 F F F 10 200-219 80 80 80 138 138 138 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 80 80 80 137 137 137 T T T 13 260-279 80 80 80 138 138 138 F F F 14 280-299 78 78 78 136 136 136 F F F 15 300-319 80 80 80 138 138 138 F F F 16 320-339 80 80 80 138 138 138 F F F 17 340-359 76 76 76 134 134 134 F F F 18 360-379 78 78 78 136 136 136 F F F 19 380-399 78 78 78 136 136 136 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 80 80 80 137 137 137 T T T 22 440-459 75 75 75 131 131 131 T T T 23 460-479 78 78 78 136 136 136 F F F 24 480-499 78 78 78 136 136 136 F F F 25 500-519 76 76 76 134 134 134 T T T 26 520-539 78 78 78 136 136 136 F F F 27 540-559 78 78 78 136 136 136 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 80, Max flush: 138 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 139 139 139 F F F 1 020-039 81 81 81 139 139 139 F F F 2 040-059 76 76 76 132 132 132 F F F 3 060-079 76 76 76 132 132 132 F F F 4 080-099 74 74 74 130 130 130 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 81 81 81 139 139 139 F F F 7 140-159 76 76 76 134 134 134 F F F 8 160-179 81 81 81 139 139 139 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 80 80 80 138 138 138 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 81 81 81 139 139 139 F F F 13 260-279 80 80 80 138 138 138 F F F 14 280-299 78 78 78 136 136 136 F F F 15 300-319 80 80 80 138 138 138 F F F 16 320-339 80 80 80 138 138 138 F F F 17 340-359 76 76 76 134 134 134 F F F 18 360-379 78 78 78 136 136 136 F F F 19 380-399 78 78 78 136 136 136 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 81 81 81 139 139 139 F F F 22 440-459 81 81 81 139 139 139 F F F 23 460-479 78 78 78 136 136 136 F F F 24 480-499 78 78 78 136 136 136 F F F 25 500-519 81 81 81 139 139 139 F F F 26 520-539 78 78 78 136 136 136 F F F 27 540-559 78 78 78 136 136 136 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 81, Max flush: 139 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.091709571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.136277747Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.136854793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.136893693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.136912662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.136930332Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} {"msg":"Crucible c62dcd4a-b39e-4941-aa99-f8ed916b19cc has session id: ace6d52f-4cc7-43cc-bd53-81c89c79f257","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.137398069Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Upstairs opts: Upstairs UUID: c62dcd4a-b39e-4941-aa99-f8ed916b19cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.137439358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Crucible stats registered with UUID: c62dcd4a-b39e-4941-aa99-f8ed916b19cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.137464178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:58.137487648Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138022104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138060604Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} {"msg":"c62dcd4a-b39e-4941-aa99-f8ed916b19cc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138090714Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138105784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"0","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138131533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"0","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138150363Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"1","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138165863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"1","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138179253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"2","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138193283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"2","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138206423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138222523Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138247503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"ds_connection connected from Ok(127.0.0.1:47622)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138283682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"ds_connection connected from Ok(127.0.0.1:34211)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138330862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"ds_connection connected from Ok(127.0.0.1:39503)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.13856413Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138916307Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"2","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.138956597Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"0","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.139311575Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"1","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.139552703Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"2","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.139684602Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"0","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.1399387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"1","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140124448Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140214178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[0]R flush_numbers[0..12]: [139, 139, 132, 132, 130, 139, 139, 134, 139, 133, 138, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140454026Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[0]R generation[0..12]: [81, 81, 76, 76, 74, 81, 81, 76, 81, 76, 80, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140472816Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140487006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[1]R flush_numbers[0..12]: [139, 139, 132, 132, 130, 139, 139, 134, 139, 133, 138, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140500966Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[1]R generation[0..12]: [81, 81, 76, 76, 74, 81, 81, 76, 81, 76, 80, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140514335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140528255Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[2]R flush_numbers[0..12]: [139, 139, 132, 132, 130, 139, 139, 134, 139, 133, 138, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140541915Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[2]R generation[0..12]: [81, 81, 76, 76, 74, 81, 81, 76, 81, 76, 80, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140555525Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140569485Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140582245Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140599095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140612355Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140625195Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140644425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140656234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"0","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140669344Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"1","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140682314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"client":"2","":"downstairs","session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140695364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140707424Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"c62dcd4a-b39e-4941-aa99-f8ed916b19cc is now active with session: ace6d52f-4cc7-43cc-bd53-81c89c79f257","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140720014Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140731874Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520,"session_id":"ace6d52f-4cc7-43cc-bd53-81c89c79f257"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140744044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} {"msg":"Activated sub_volume c62dcd4a-b39e-4941-aa99-f8ed916b19cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.140755064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 374 len 20480 data: 8 7 6 5 8 02/30 Write block 77 len 24576 data: 7 8 8 6 6 5 03/30 Read block 193 len 40960 04/30 Read block 274 len 16384 05/30 Read block 268 len 40960 06/30 Read block 542 len 8192 07/30 Write block 127 len 12288 data: 8 6 7 08/30 Read block 71 len 28672 09/30 Read block 212 len 36864 10/30 Read block 263 len 40960 11/30 Write block 241 len 16384 data: 6 5 5 4 12/30 Flush 13/30 Write block 519 len 16384 data: 10 9 12 13 14/30 Read block 39 len 12288 15/30 Write block 18 len 36864 data: 8 10 8 9 8 11 10 10 11 16/30 Read block 267 len 4096 17/30 Read block 583 len 24576 18/30 Write block 319 len 20480 data: 8 9 11 7 8 19/30 Write block 147 len 28672 data: 2 4 5 6 7 8 7 20/30 Write block 342 len 28672 data: 4 5 5 5 4 7 6 21/30 Read block 150 len 8192 22/30 Flush 23/30 Write block 47 len 40960 data: 5 4 5 4 5 4 5 6 9 9 24/30 Read block 188 len 40960 25/30 Read block 331 len 36864 26/30 Read block 127 len 8192 27/30 Write block 474 len 8192 data: 4 3 28/30 Write block 203 len 4096 data: 5 29/30 Write block 386 len 4096 data: 8 30/30 Write block 248 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 2 Done Sent Sent false 1027 Acked Write 1 Done Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 4 4 10 Done 6 4 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:58.160420456Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 141 141 141 F F F 1 020-039 82 82 82 141 141 141 F F F 2 040-059 76 76 76 132 132 132 T T T 3 060-079 82 82 82 140 140 140 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 82 82 82 140 140 140 F F F 7 140-159 82 82 82 141 141 141 F F F 8 160-179 81 81 81 139 139 139 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 80 80 80 138 138 138 T T T 11 220-239 70 70 70 123 123 123 F F F 12 240-259 82 82 82 140 140 140 T T T 13 260-279 80 80 80 138 138 138 F F F 14 280-299 78 78 78 136 136 136 F F F 15 300-319 82 82 82 141 141 141 F F F 16 320-339 82 82 82 141 141 141 F F F 17 340-359 82 82 82 141 141 141 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 78 78 78 136 136 136 T T T 20 400-419 78 78 78 136 136 136 F F F 21 420-439 81 81 81 139 139 139 F F F 22 440-459 81 81 81 139 139 139 F F F 23 460-479 78 78 78 136 136 136 T T T 24 480-499 78 78 78 136 136 136 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 78 78 78 136 136 136 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 82, Max flush: 141 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 141 141 141 F F F 1 020-039 82 82 82 141 141 141 F F F 2 040-059 83 83 83 142 142 142 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 82 82 82 140 140 140 F F F 7 140-159 82 82 82 141 141 141 F F F 8 160-179 81 81 81 139 139 139 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 83 83 83 142 142 142 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 80 80 80 138 138 138 F F F 14 280-299 78 78 78 136 136 136 F F F 15 300-319 82 82 82 141 141 141 F F F 16 320-339 82 82 82 141 141 141 F F F 17 340-359 82 82 82 141 141 141 F F F 18 360-379 82 82 82 140 140 140 F F F 19 380-399 83 83 83 142 142 142 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 81 81 81 139 139 139 F F F 22 440-459 81 81 81 139 139 139 F F F 23 460-479 83 83 83 142 142 142 F F F 24 480-499 78 78 78 136 136 136 F F F 25 500-519 82 82 82 141 141 141 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 78 78 78 136 136 136 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 83, Max flush: 142 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.393608377Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:02.436849743Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.437241621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.469168632Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.469734947Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.469777237Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.469794447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.469812397Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} {"msg":"Crucible 19a66954-5d6c-4521-95cf-cde5a4808ad1 has session id: f577f31b-7f16-4227-b6ab-8c8b51e6006a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.470418212Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Upstairs opts: Upstairs UUID: 19a66954-5d6c-4521-95cf-cde5a4808ad1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.470459922Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Crucible stats registered with UUID: 19a66954-5d6c-4521-95cf-cde5a4808ad1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.470485492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:02.470505022Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.470937408Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.470972078Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} {"msg":"19a66954-5d6c-4521-95cf-cde5a4808ad1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471028838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471055827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"0","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471083897Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"0","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471097977Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"1","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471115987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"1","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471140387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"2","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471161577Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"2","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471182906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471202956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471227236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48443)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471258646Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56045)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471290126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45886)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.471318665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.47199519Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"1","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.47203146Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"2","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.472175749Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"0","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.472716715Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"1","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.472755705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"2","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.472785885Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"0","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.47333619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.47336185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[0]R flush_numbers[0..12]: [141, 141, 142, 140, 140, 139, 140, 141, 139, 133, 142, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.47341958Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[0]R generation[0..12]: [82, 82, 83, 82, 82, 81, 82, 82, 81, 76, 83, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473439949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473454989Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[1]R flush_numbers[0..12]: [141, 141, 142, 140, 140, 139, 140, 141, 139, 133, 142, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473472669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[1]R generation[0..12]: [82, 82, 83, 82, 82, 81, 82, 82, 81, 76, 83, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473486609Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473500469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[2]R flush_numbers[0..12]: [141, 141, 142, 140, 140, 139, 140, 141, 139, 133, 142, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473514279Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[2]R generation[0..12]: [82, 82, 83, 82, 82, 81, 82, 82, 81, 76, 83, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473527629Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473541239Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473553939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473566709Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473579308Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473592128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"":"downstairs","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:02.473604948Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473618998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"0","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473632258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"1","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473645238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"client":"2","":"downstairs","session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473659898Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473672158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"19a66954-5d6c-4521-95cf-cde5a4808ad1 is now active with session: f577f31b-7f16-4227-b6ab-8c8b51e6006a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473684798Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473696648Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528,"session_id":"f577f31b-7f16-4227-b6ab-8c8b51e6006a"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473708618Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} {"msg":"Activated sub_volume 19a66954-5d6c-4521-95cf-cde5a4808ad1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.473719878Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 359 len 28672 data: 9 7 5 4 4 5 5 03/30 Read block 243 len 32768 04/30 Read block 39 len 12288 05/30 Write block 421 len 28672 data: 7 7 8 10 8 6 6 06/30 Read block 571 len 32768 07/30 Write block 16 len 8192 data: 9 10 08/30 Write block 547 len 40960 data: 12 9 8 7 4 8 8 7 9 10 09/30 Read block 590 len 28672 10/30 Write block 190 len 12288 data: 6 6 7 11/30 Flush 12/30 Write block 186 len 8192 data: 7 6 13/30 Write block 293 len 32768 data: 14 9 5 4 4 4 6 8 14/30 Read block 377 len 36864 15/30 Read block 117 len 16384 16/30 Read block 521 len 32768 17/30 Read block 582 len 16384 18/30 Read block 325 len 40960 19/30 Write block 261 len 40960 data: 6 6 9 8 10 10 10 10 9 9 20/30 Read block 360 len 36864 21/30 Read block 385 len 36864 22/30 Write block 492 len 40960 data: 10 9 9 9 6 8 9 9 8 7 23/30 Read block 50 len 40960 24/30 Write block 280 len 12288 data: 5 6 5 25/30 Read block 416 len 36864 26/30 Write block 131 len 4096 data: 12 27/30 Write block 439 len 8192 data: 10 7 28/30 Write block 204 len 24576 data: 5 6 6 7 7 6 29/30 Read block 313 len 28672 30/30 Read block 466 len 4096 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 2 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:02.495631093Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 144 144 144 F F F 1 020-039 82 82 82 141 141 141 F F F 2 040-059 83 83 83 142 142 142 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 82 82 82 140 140 140 T T T 7 140-159 82 82 82 141 141 141 F F F 8 160-179 81 81 81 139 139 139 F F F 9 180-199 84 84 84 144 144 144 T T T 10 200-219 83 83 83 142 142 142 T T T 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 80 80 80 138 138 138 T T T 14 280-299 78 78 78 136 136 136 T T T 15 300-319 82 82 82 141 141 141 T T T 16 320-339 82 82 82 141 141 141 F F F 17 340-359 84 84 84 144 144 144 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 83 83 83 142 142 142 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 84 84 84 144 144 144 T T T 22 440-459 81 81 81 139 139 139 T T T 23 460-479 83 83 83 142 142 142 F F F 24 480-499 78 78 78 136 136 136 T T T 25 500-519 82 82 82 141 141 141 T T T 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 144 144 144 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 84, Max flush: 144 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 144 144 144 F F F 1 020-039 82 82 82 141 141 141 F F F 2 040-059 83 83 83 142 142 142 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 82 82 82 140 140 140 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 85 85 85 145 145 145 F F F 7 140-159 82 82 82 141 141 141 F F F 8 160-179 81 81 81 139 139 139 F F F 9 180-199 85 85 85 145 145 145 F F F 10 200-219 85 85 85 145 145 145 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 85 85 85 145 145 145 F F F 14 280-299 85 85 85 145 145 145 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 82 82 82 141 141 141 F F F 17 340-359 84 84 84 144 144 144 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 83 83 83 142 142 142 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 85 85 85 145 145 145 F F F 23 460-479 83 83 83 142 142 142 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 84 84 84 144 144 144 F F F 28 560-579 80 80 80 137 137 137 F F F 29 580-599 73 73 73 127 127 127 F F F Max gen: 85, Max flush: 145 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.786136095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.830517373Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.831147718Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.831191828Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.831207268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.831221997Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} {"msg":"Crucible 1df9028e-f9d6-4934-93d6-e4c4065afb9a has session id: 25fca747-b0e4-4c2a-b76d-fef1deb5ae4a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.831817783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Upstairs opts: Upstairs UUID: 1df9028e-f9d6-4934-93d6-e4c4065afb9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.831857193Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Crucible stats registered with UUID: 1df9028e-f9d6-4934-93d6-e4c4065afb9a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.831881032Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:06.831901462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832471748Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832498588Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} {"msg":"1df9028e-f9d6-4934-93d6-e4c4065afb9a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832557507Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832575217Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"0","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832607487Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"0","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832621927Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"1","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832636857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"1","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832660607Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"2","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832687877Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"2","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832713216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832739046Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832762906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37147)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832788346Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60864)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832815776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63243)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.832840725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.833461661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"1","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83349207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"0","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.834169835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"1","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.834218155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"0","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.834817361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83485328Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83488037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"2","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.835428846Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"2","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[0]R flush_numbers[0..12]: [144, 141, 142, 140, 140, 139, 145, 141, 139, 145, 145, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.835940822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[0]R generation[0..12]: [84, 82, 83, 82, 82, 81, 85, 82, 81, 85, 85, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.835971532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.835991272Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[1]R flush_numbers[0..12]: [144, 141, 142, 140, 140, 139, 145, 141, 139, 145, 145, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836009092Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[1]R generation[0..12]: [84, 82, 83, 82, 82, 81, 85, 82, 81, 85, 85, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836026252Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836043551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[2]R flush_numbers[0..12]: [144, 141, 142, 140, 140, 139, 145, 141, 139, 145, 145, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836060741Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[2]R generation[0..12]: [84, 82, 83, 82, 82, 81, 85, 82, 81, 85, 85, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836077601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836095001Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836116171Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836134181Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83615048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83616671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"":"downstairs","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83618276Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83619763Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"0","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83621392Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"1","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83623779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"client":"2","":"downstairs","session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83625425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83626918Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"1df9028e-f9d6-4934-93d6-e4c4065afb9a is now active with session: 25fca747-b0e4-4c2a-b76d-fef1deb5ae4a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.83628513Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836299769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536,"session_id":"25fca747-b0e4-4c2a-b76d-fef1deb5ae4a"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836315059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} {"msg":"Activated sub_volume 1df9028e-f9d6-4934-93d6-e4c4065afb9a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:06.836329289Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 295 len 20480 data: 6 5 5 5 7 02/30 Write block 469 len 28672 data: 6 7 5 4 4 5 4 03/30 Read block 367 len 12288 04/30 Write block 83 len 32768 data: 4 6 6 6 7 7 8 10 05/30 Write block 6 len 36864 data: 7 9 9 10 9 11 9 8 12 06/30 Write block 544 len 28672 data: 8 9 10 13 10 9 8 07/30 Write block 279 len 8192 data: 5 6 08/30 Read block 277 len 36864 09/30 Write block 553 len 36864 data: 9 8 10 11 12 10 9 9 9 10/30 Write block 163 len 8192 data: 7 6 11/30 Flush 12/30 Write block 544 len 24576 data: 9 10 11 14 11 10 13/30 Read block 516 len 28672 14/30 Read block 103 len 20480 15/30 Read block 573 len 20480 16/30 Read block 586 len 12288 17/30 Read block 553 len 16384 18/30 Read block 361 len 24576 19/30 Write block 586 len 12288 data: 6 7 7 20/30 Read block 171 len 20480 21/30 Flush 22/30 Flush 23/30 Write block 389 len 28672 data: 5 4 4 4 3 2 2 24/30 Write block 139 len 32768 data: 5 8 5 5 5 3 3 3 25/30 Write block 476 len 4096 data: 3 26/30 Flush 27/30 Write block 4 len 32768 data: 6 7 8 10 10 11 10 12 28/30 Write block 470 len 16384 data: 8 6 5 5 29/30 Read block 426 len 24576 30/30 Write block 595 len 8192 data: 8 7 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:06.855950762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 146 146 146 T T T 1 020-039 82 82 82 141 141 141 F F F 2 040-059 83 83 83 142 142 142 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 86 86 86 146 146 146 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 86 86 86 149 149 149 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 85 85 85 145 145 145 F F F 10 200-219 85 85 85 145 145 145 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 86 86 86 146 146 146 F F F 14 280-299 86 86 86 146 146 146 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 82 82 82 141 141 141 F F F 17 340-359 84 84 84 144 144 144 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 85 85 85 145 145 145 F F F 23 460-479 86 86 86 149 149 149 T T T 24 480-499 85 85 85 145 145 145 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 86 86 86 147 147 147 F F F 28 560-579 86 86 86 146 146 146 F F F 29 580-599 86 86 86 147 147 147 T T T Max gen: 86, Max flush: 149 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 150 150 150 F F F 1 020-039 82 82 82 141 141 141 F F F 2 040-059 83 83 83 142 142 142 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 86 86 86 146 146 146 F F F 5 100-119 81 81 81 139 139 139 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 86 86 86 149 149 149 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 85 85 85 145 145 145 F F F 10 200-219 85 85 85 145 145 145 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 86 86 86 146 146 146 F F F 14 280-299 86 86 86 146 146 146 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 82 82 82 141 141 141 F F F 17 340-359 84 84 84 144 144 144 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 85 85 85 145 145 145 F F F 23 460-479 87 87 87 150 150 150 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 86 86 86 147 147 147 F F F 28 560-579 86 86 86 146 146 146 F F F 29 580-599 87 87 87 150 150 150 F F F Max gen: 87, Max flush: 150 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.084900015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.129289563Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.130101206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.130145086Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.130161006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.130176666Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} {"msg":"Crucible 6c31ad1d-63d2-479c-a0f0-dcb4aab38beb has session id: 2cc1fa2d-0b8d-4d90-8565-a2efc5578f81","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.130836301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Upstairs opts: Upstairs UUID: 6c31ad1d-63d2-479c-a0f0-dcb4aab38beb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13092751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Crucible stats registered with UUID: 6c31ad1d-63d2-479c-a0f0-dcb4aab38beb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13096945Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:11.13099227Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131464156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131488456Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} {"msg":"6c31ad1d-63d2-479c-a0f0-dcb4aab38beb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131584545Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131602155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"0","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131630155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"0","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131644385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"1","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131658555Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"1","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131671585Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"2","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131685515Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"2","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131698114Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131711514Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131725534Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:51921)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131862833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:57221)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.131905673Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:51078)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.132045032Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.132671277Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"2","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.132711517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"0","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.133353882Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"0","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.133457421Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"1","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.133517721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"2","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.133964188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134065647Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"1","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134089207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[0]R flush_numbers[0..12]: [150, 141, 142, 140, 146, 139, 149, 149, 146, 145, 145, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134667442Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[0]R generation[0..12]: [87, 82, 83, 82, 86, 81, 86, 86, 86, 85, 85, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134694612Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134710252Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[1]R flush_numbers[0..12]: [150, 141, 142, 140, 146, 139, 149, 149, 146, 145, 145, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134724392Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[1]R generation[0..12]: [87, 82, 83, 82, 86, 81, 86, 86, 86, 85, 85, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134741782Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134756402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[2]R flush_numbers[0..12]: [150, 141, 142, 140, 146, 139, 149, 149, 146, 145, 145, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134769962Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[2]R generation[0..12]: [87, 82, 83, 82, 86, 81, 86, 86, 86, 85, 85, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134783331Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134797061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134809871Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134822951Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"}Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134842741Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134855531Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"":"downstairs","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134868601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.134880511Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"0","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13489357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"1","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13490657Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"client":"2","":"downstairs","session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13491961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13493171Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"6c31ad1d-63d2-479c-a0f0-dcb4aab38beb is now active with session: 2cc1fa2d-0b8d-4d90-8565-a2efc5578f81","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13494692Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13495919Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544,"session_id":"2cc1fa2d-0b8d-4d90-8565-a2efc5578f81"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13497145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} {"msg":"Activated sub_volume 6c31ad1d-63d2-479c-a0f0-dcb4aab38beb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:11.13498263Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 206 len 40960 data: 7 8 8 7 8 8 7 8 10 9 02/30 Read block 278 len 36864 03/30 Write block 582 len 20480 data: 5 5 6 6 7 04/30 Write block 337 len 20480 data: 8 9 7 6 4 05/30 Read block 187 len 16384 06/30 Write block 200 len 4096 data: 6 07/30 Read block 376 len 32768 08/30 Read block 425 len 24576 09/30 Read block 5 len 4096 10/30 Write block 299 len 4096 data: 8 11/30 Write block 320 len 36864 data: 10 12 8 9 11 11 12 12 9 12/30 Write block 464 len 16384 data: 7 8 6 6 13/30 Read block 250 len 36864 14/30 Read block 473 len 4096 15/30 Read block 550 len 28672 16/30 Read block 414 len 20480 17/30 Read block 79 len 24576 18/30 Write block 115 len 8192 data: 6 8 19/30 Read block 328 len 36864 20/30 Flush 21/30 Write block 146 len 28672 data: 4 3 5 6 7 8 9 22/30 Write block 349 len 16384 data: 8 8 11 11 23/30 Read block 361 len 32768 24/30 Read block 351 len 40960 25/30 Flush 26/30 Write block 38 len 16384 data: 12 12 8 7 27/30 Read block 233 len 20480 28/30 Read block 485 len 8192 29/30 Read block 483 len 24576 30/30 Write block 577 len 36864 data: 8 7 7 7 4 6 6 7 7 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:11.155075179Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 150 150 150 F F F 1 020-039 82 82 82 141 141 141 T T T 2 040-059 83 83 83 142 142 142 T T T 3 060-079 82 82 82 140 140 140 F F F 4 080-099 86 86 86 146 146 146 F F F 5 100-119 88 88 88 151 151 151 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 88 88 88 152 152 152 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 85 85 85 145 145 145 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 86 86 86 146 146 146 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 88 88 88 151 151 151 F F F 17 340-359 88 88 88 152 152 152 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 85 85 85 145 145 145 F F F 23 460-479 88 88 88 151 151 151 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 86 86 86 147 147 147 F F F 28 560-579 86 86 86 146 146 146 T T T 29 580-599 88 88 88 151 151 151 T T T Max gen: 88, Max flush: 152 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 150 150 150 F F F 1 020-039 89 89 89 153 153 153 F F F 2 040-059 89 89 89 153 153 153 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 86 86 86 146 146 146 F F F 5 100-119 88 88 88 151 151 151 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 88 88 88 152 152 152 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 85 85 85 145 145 145 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 86 86 86 146 146 146 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 88 88 88 151 151 151 F F F 17 340-359 88 88 88 152 152 152 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 85 85 85 145 145 145 F F F 23 460-479 88 88 88 151 151 151 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 82 82 82 141 141 141 F F F 27 540-559 86 86 86 147 147 147 F F F 28 560-579 89 89 89 153 153 153 F F F 29 580-599 89 89 89 153 153 153 F F F Max gen: 89, Max flush: 153 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.385724659Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.429504502Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.430089127Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.430131797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.430151507Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.430166817Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552} {"msg":"Crucible 74e8189a-df06-41c7-b169-3aea7cf88a66 has session id: 79f6e95a-3505-4d8e-9412-f15671b820fc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.430761982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Upstairs opts: Upstairs UUID: 74e8189a-df06-41c7-b169-3aea7cf88a66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.430792622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Crucible stats registered with UUID: 74e8189a-df06-41c7-b169-3aea7cf88a66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.430807132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:15.430819732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431316238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431349378Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552} {"msg":"74e8189a-df06-41c7-b169-3aea7cf88a66 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431407797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431438117Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"0","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431485817Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"0","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431517847Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"1","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431547436Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"1","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431575376Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431607686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431680325Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431710915Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431740925Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:62492)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431827594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:15.431860184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:15.431877724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45269)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431901004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:15.431918104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.431931273Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.432247081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"0","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.43241036Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"1","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.432838297Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"0","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.432986005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"1","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.433433122Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:15.433518941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.432942622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47703)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.433112501Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.434058054Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.434632399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[0]R flush_numbers[0..12]: [150, 153, 153, 140, 146, 151, 149, 152, 146, 145, 151, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435270624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[0]R generation[0..12]: [87, 89, 89, 82, 86, 88, 86, 88, 86, 85, 88, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435296604Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435312294Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[1]R flush_numbers[0..12]: [150, 153, 153, 140, 146, 151, 149, 152, 146, 145, 151, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435326544Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[1]R generation[0..12]: [87, 89, 89, 82, 86, 88, 86, 88, 86, 85, 88, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435340264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435354234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[2]R flush_numbers[0..12]: [150, 153, 153, 140, 146, 151, 149, 152, 146, 145, 151, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435367804Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[2]R generation[0..12]: [87, 89, 89, 82, 86, 88, 86, 88, 86, 85, 88, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435381044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435398554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435411674Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435424703Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435437473Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435450293Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"":"downstairs","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435463203Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435475193Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"0","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435488473Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"1","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435501483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"client":"2","":"downstairs","session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435514383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435526423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"74e8189a-df06-41c7-b169-3aea7cf88a66 is now active with session: 79f6e95a-3505-4d8e-9412-f15671b820fc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435539002Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435550893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552,"session_id":"79f6e95a-3505-4d8e-9412-f15671b820fc"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435562962Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552} {"msg":"Activated sub_volume 74e8189a-df06-41c7-b169-3aea7cf88a66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:25.435574642Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":Wait for a query_work_queue command to finish before sending IO 1552} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 34 len 40960 data: 10 10 12 11 13 13 9 8 5 6 03/30 Write block 555 len 36864 data: 11 12 13 11 10 10 10 8 7 04/30 Write block 346 len 28672 data: 5 8 7 9 9 12 12 05/30 Read block 246 len 4096 06/30 Write block 549 len 40960 data: 11 9 5 9 10 9 12 13 14 12 07/30 Read block 545 len 32768 08/30 Write block 94 len 28672 data: 9 8 7 5 4 6 6 09/30 Read block 155 len 16384 10/30 Write block 327 len 4096 data: 13 11/30 Flush 12/30 Write block 447 len 8192 data: 6 7 13/30 Read block 331 len 32768 14/30 Read block 182 len 32768 15/30 Flush 16/30 Write block 520 len 8192 data: 10 13 17/30 Read block 184 len 4096 18/30 Write block 90 len 28672 data: 11 9 11 10 10 9 8 19/30 Read block 454 len 4096 20/30 Write block 574 len 40960 data: 5 7 8 9 8 8 8 5 7 7 21/30 Read block 229 len 36864 22/30 Read block 201 len 20480 23/30 Read block 88 len 28672 24/30 Read block 291 len 40960 25/30 Read block 299 len 32768 26/30 Flush 27/30 Write block 341 len 28672 data: 5 5 6 6 6 6 9 28/30 Read block 243 len 12288 29/30 Read block 346 len 28672 30/30 Read block 218 len 8192 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:25.457401449Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 150 150 150 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 90 90 90 157 157 157 F F F 5 100-119 90 90 90 155 155 155 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 88 88 88 152 152 152 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 85 85 85 145 145 145 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 86 86 86 146 146 146 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 90 90 90 155 155 155 F F F 17 340-359 90 90 90 155 155 155 T T T 18 360-379 84 84 84 144 144 144 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 90 90 90 156 156 156 F F F 23 460-479 88 88 88 151 151 151 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 90 90 90 155 155 155 F F F 28 560-579 90 90 90 157 157 157 F F F 29 580-599 90 90 90 157 157 157 F F F Max gen: 90, Max flush: 157 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 150 150 150 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 82 82 82 140 140 140 F F F 4 080-099 90 90 90 157 157 157 F F F 5 100-119 90 90 90 155 155 155 F F F 6 120-139 86 86 86 149 149 149 F F F 7 140-159 88 88 88 152 152 152 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 85 85 85 145 145 145 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 83 83 83 142 142 142 F F F 13 260-279 86 86 86 146 146 146 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 90 90 90 155 155 155 F F F 17 340-359 91 91 91 158 158 158 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 86 86 86 149 149 149 F F F 20 400-419 78 78 78 136 136 136 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 90 90 90 156 156 156 F F F 23 460-479 88 88 88 151 151 151 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 90 90 90 155 155 155 F F F 28 560-579 90 90 90 157 157 157 F F F 29 580-599 90 90 90 157 157 157 F F F Max gen: 91, Max flush: 158 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.674151464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.718133644Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.718809399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.718861599Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.718881319Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.718899768Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} {"msg":"Crucible 3e039461-755d-4f8f-a67d-44cba5991d86 has session id: 67bdd64e-6704-4da7-9eea-65c1399961dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.719496274Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Upstairs opts: Upstairs UUID: 3e039461-755d-4f8f-a67d-44cba5991d86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.719601073Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Crucible stats registered with UUID: 3e039461-755d-4f8f-a67d-44cba5991d86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.719631013Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:29.719652053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720134739Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720159909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} {"msg":"3e039461-755d-4f8f-a67d-44cba5991d86 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720253778Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720272198Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"0","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720298948Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"0","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720312808Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"1","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720326848Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"1","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720348288Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"2","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720375548Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"2","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720401157Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720428587Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720453247Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:61236)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720482427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38795)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720547786Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45282)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.720682045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.721130352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"0","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.721167942Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"2","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.721798977Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"0","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.721924706Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"2","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.722062405Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"1","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.722378353Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.722554011Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.72272681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"1","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[0]R flush_numbers[0..12]: [150, 155, 155, 140, 157, 155, 149, 152, 146, 145, 151, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723247876Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[0]R generation[0..12]: [87, 90, 90, 82, 90, 90, 86, 88, 86, 85, 88, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723273716Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723292116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[1]R flush_numbers[0..12]: [150, 155, 155, 140, 157, 155, 149, 152, 146, 145, 151, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723312686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[1]R generation[0..12]: [87, 90, 90, 82, 90, 90, 86, 88, 86, 85, 88, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723329755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723346895Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[2]R flush_numbers[0..12]: [150, 155, 155, 140, 157, 155, 149, 152, 146, 145, 151, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723363505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[2]R generation[0..12]: [87, 90, 90, 82, 90, 90, 86, 88, 86, 85, 88, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723380005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723396955Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Max found gen is 92","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-03-18T05:04:29.723419315Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723435705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723451554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723467504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"":"downstairs","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723483474Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723498184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"0","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723514324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723533864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"client":"2","":"downstairs","session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723550024Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723567494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"3e039461-755d-4f8f-a67d-44cba5991d86 is now active with session: 67bdd64e-6704-4da7-9eea-65c1399961dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723583324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723597973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560,"session_id":"67bdd64e-6704-4da7-9eea-65c1399961dc"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723613073Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} {"msg":"Activated sub_volume 3e039461-755d-4f8f-a67d-44cba5991d86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:29.723626753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 258 len 24576 data: 6 5 5 7 7 10 02/30 Read block 530 len 40960 03/30 Read block 580 len 8192 04/30 Write block 93 len 8192 data: 11 11 05/30 Flush 06/30 Write block 150 len 16384 data: 8 9 10 8 07/30 Flush 08/30 Write block 456 len 16384 data: 8 8 4 4 09/30 Read block 186 len 12288 10/30 Read block 461 len 36864 11/30 Read block 139 len 28672 12/30 Read block 472 len 36864 13/30 Read block 337 len 40960 14/30 Read block 526 len 32768 15/30 Flush 16/30 Write block 208 len 20480 data: 9 8 9 9 8 17/30 Write block 206 len 28672 data: 8 9 10 9 10 10 9 18/30 Read block 7 len 4096 19/30 Read block 566 len 28672 20/30 Write block 121 len 20480 data: 7 7 8 10 9 21/30 Write block 66 len 32768 data: 6 6 5 6 6 6 7 7 22/30 Read block 272 len 32768 23/30 Write block 180 len 20480 data: 8 7 8 8 8 24/30 Write block 248 len 32768 data: 8 8 7 6 5 6 9 8 25/30 Write block 491 len 12288 data: 9 11 10 26/30 Read block 59 len 32768 27/30 Write block 418 len 24576 data: 3 5 5 8 8 9 28/30 Write block 580 len 40960 data: 9 6 8 8 8 8 8 8 8 7 29/30 Read block 79 len 16384 30/30 Write block 384 len 36864 data: 6 7 9 6 5 6 5 5 5 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 5 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:29.74410824Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 150 150 150 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 82 82 82 140 140 140 T T T 4 080-099 92 92 92 159 159 159 F F F 5 100-119 90 90 90 155 155 155 F F F 6 120-139 86 86 86 149 149 149 T T T 7 140-159 92 92 92 160 160 160 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 85 85 85 145 145 145 T T T 10 200-219 88 88 88 151 151 151 T T T 11 220-239 70 70 70 123 123 123 F F F 12 240-259 92 92 92 159 159 159 T T T 13 260-279 92 92 92 159 159 159 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 90 90 90 155 155 155 F F F 17 340-359 91 91 91 158 158 158 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 86 86 86 149 149 149 T T T 20 400-419 78 78 78 136 136 136 T T T 21 420-439 85 85 85 145 145 145 T T T 22 440-459 92 92 92 161 161 161 F F F 23 460-479 88 88 88 151 151 151 F F F 24 480-499 85 85 85 145 145 145 T T T 25 500-519 85 85 85 145 145 145 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 90 90 90 155 155 155 F F F 28 560-579 90 90 90 157 157 157 F F F 29 580-599 90 90 90 157 157 157 T T T Max gen: 92, Max flush: 161 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 150 150 150 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 93 93 93 162 162 162 F F F 4 080-099 92 92 92 159 159 159 F F F 5 100-119 90 90 90 155 155 155 F F F 6 120-139 93 93 93 162 162 162 F F F 7 140-159 92 92 92 160 160 160 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 93 93 93 162 162 162 F F F 11 220-239 70 70 70 123 123 123 F F F 12 240-259 93 93 93 162 162 162 F F F 13 260-279 92 92 92 159 159 159 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 90 90 90 155 155 155 F F F 17 340-359 91 91 91 158 158 158 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 93 93 93 162 162 162 F F F 21 420-439 93 93 93 162 162 162 F F F 22 440-459 92 92 92 161 161 161 F F F 23 460-479 88 88 88 151 151 151 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 85 85 85 145 145 145 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 90 90 90 155 155 155 F F F 28 560-579 90 90 90 157 157 157 F F F 29 580-599 93 93 93 162 162 162 F F F Max gen: 93, Max flush: 162 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.011582953Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.056229469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.056846884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.056899154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.056917714Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.056935734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} {"msg":"Crucible 61bbb6b9-d16e-4189-b73a-ab8c38e6da3a has session id: cbcb1fc2-d483-423f-ac46-830b57483706","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.05739572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Upstairs opts: Upstairs UUID: 61bbb6b9-d16e-4189-b73a-ab8c38e6da3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.05744184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Crucible stats registered with UUID: 61bbb6b9-d16e-4189-b73a-ab8c38e6da3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.05746895Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:34.057491159Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058027295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058061225Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} {"msg":"61bbb6b9-d16e-4189-b73a-ab8c38e6da3a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058124345Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058142045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"0","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058168774Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"0","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058187404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"1","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058211864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"1","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058239074Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"2","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058265684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"2","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058284293Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058302133Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058328223Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"ds_connection connected from Ok(127.0.0.1:34906)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058355963Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"ds_connection connected from Ok(127.0.0.1:61041)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058383843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"ds_connection connected from Ok(127.0.0.1:49025)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058610841Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.058987718Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"1","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.059167647Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"0","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.059253896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"2","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.059599383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"1","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.059933851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"0","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.059962831Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"2","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060305278Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060533727Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[0]R flush_numbers[0..12]: [150, 155, 155, 162, 159, 155, 162, 160, 146, 162, 162, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060663475Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[0]R generation[0..12]: [87, 90, 90, 93, 92, 90, 93, 92, 86, 93, 93, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060684285Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060699125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[1]R flush_numbers[0..12]: [150, 155, 155, 162, 159, 155, 162, 160, 146, 162, 162, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060712965Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[1]R generation[0..12]: [87, 90, 90, 93, 92, 90, 93, 92, 86, 93, 93, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060726725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060740735Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[2]R flush_numbers[0..12]: [150, 155, 155, 162, 159, 155, 162, 160, 146, 162, 162, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060754595Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[2]R generation[0..12]: [87, 90, 90, 93, 92, 90, 93, 92, 86, 93, 93, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060768075Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060782035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060794974Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060812505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060826154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:34.060839054Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"":"downstairs","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060855654Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060867464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"0","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060880684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"1","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060893744Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"client":"2","":"downstairs","session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060906804Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060919074Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"61bbb6b9-d16e-4189-b73a-ab8c38e6da3a is now active with session: cbcb1fc2-d483-423f-ac46-830b57483706","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060931833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060943804Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568,"session_id":"cbcb1fc2-d483-423f-ac46-830b57483706"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060955913Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} {"msg":"Activated sub_volume 61bbb6b9-d16e-4189-b73a-ab8c38e6da3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:34.060967313Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 109 len 8192 data: 7 5 02/30 Read block 229 len 20480 03/30 Write block 211 len 8192 data: 11 10 04/30 Write block 486 len 40960 data: 11 10 9 9 8 10 12 11 10 10 05/30 Flush 06/30 Write block 9 len 40960 data: 12 11 13 10 9 13 13 10 11 9 07/30 Flush 08/30 Write block 474 len 24576 data: 6 5 4 3 3 5 09/30 Read block 441 len 20480 10/30 Read block 266 len 40960 11/30 Flush 12/30 Flush 13/30 Write block 541 len 8192 data: 8 8 14/30 Write block 443 len 28672 data: 7 7 6 8 7 8 9 15/30 Read block 94 len 36864 16/30 Read block 272 len 16384 17/30 Read block 259 len 16384 18/30 Read block 33 len 20480 19/30 Read block 415 len 4096 20/30 Read block 154 len 8192 21/30 Write block 506 len 32768 data: 11 13 11 11 11 11 8 7 22/30 Read block 329 len 8192 23/30 Write block 407 len 4096 data: 9 24/30 Write block 222 len 24576 data: 5 6 5 7 7 9 25/30 Write block 546 len 16384 data: 12 15 12 12 26/30 Flush 27/30 Write block 547 len 12288 data: 16 13 13 28/30 Read block 548 len 4096 29/30 Read block 160 len 16384 30/30 Write block 243 len 36864 data: 6 5 4 4 5 9 9 8 7 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:34.081257111Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 164 164 164 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 93 93 93 162 162 162 F F F 4 080-099 92 92 92 159 159 159 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 93 93 93 162 162 162 F F F 7 140-159 92 92 92 160 160 160 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 94 94 94 163 163 163 F F F 11 220-239 94 94 94 167 167 167 F F F 12 240-259 93 93 93 162 162 162 T T T 13 260-279 92 92 92 159 159 159 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 90 90 90 155 155 155 F F F 17 340-359 91 91 91 158 158 158 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 93 93 93 162 162 162 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 94 94 94 167 167 167 T T T 28 560-579 90 90 90 157 157 157 F F F 29 580-599 93 93 93 162 162 162 F F F Max gen: 94, Max flush: 167 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 164 164 164 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 93 93 93 162 162 162 F F F 4 080-099 92 92 92 159 159 159 F F F 5 100-119 94 94 94 163 163 163 F F F 6 120-139 93 93 93 162 162 162 F F F 7 140-159 92 92 92 160 160 160 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 94 94 94 163 163 163 F F F 11 220-239 94 94 94 167 167 167 F F F 12 240-259 95 95 95 168 168 168 F F F 13 260-279 92 92 92 159 159 159 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 90 90 90 155 155 155 F F F 17 340-359 91 91 91 158 158 158 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 93 93 93 162 162 162 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 95 95 95 168 168 168 F F F 28 560-579 90 90 90 157 157 157 F F F 29 580-599 93 93 93 162 162 162 F F F Max gen: 95, Max flush: 168 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.303385926Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:38.345722909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.346077206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.377129384Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.37768182Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.377720789Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.377735879Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.377749969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} {"msg":"Crucible 8e992ed0-307f-44e3-8ee4-58640ce6142a has session id: 4bac74bb-f9e7-439a-8d59-1e97ec36f399","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.378299385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Upstairs opts: Upstairs UUID: 8e992ed0-307f-44e3-8ee4-58640ce6142a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.378330975Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Crucible stats registered with UUID: 8e992ed0-307f-44e3-8ee4-58640ce6142a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.378349624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:38.378363064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.378833861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.37891221Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} {"msg":"8e992ed0-307f-44e3-8ee4-58640ce6142a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.37894234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.37895673Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"0","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.37898298Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"0","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.37900856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"1","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379023419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"1","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379037699Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"2","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379051849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"2","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379065299Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379079069Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379092559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"ds_connection connected from Ok(127.0.0.1:37176)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379336907Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"ds_connection connected from Ok(127.0.0.1:49002)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379384917Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"ds_connection connected from Ok(127.0.0.1:46736)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379415836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379944213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"2","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379973182Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"0","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.379988882Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"1","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.380631387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"1","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.380661097Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"2","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.380689017Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"0","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381178443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381295753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[0]R flush_numbers[0..12]: [164, 155, 155, 162, 159, 163, 162, 160, 146, 162, 163, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381368282Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[0]R generation[0..12]: [94, 90, 90, 93, 92, 94, 93, 92, 86, 93, 94, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381387932Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381402662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[1]R flush_numbers[0..12]: [164, 155, 155, 162, 159, 163, 162, 160, 146, 162, 163, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381416672Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[1]R generation[0..12]: [94, 90, 90, 93, 92, 94, 93, 92, 86, 93, 94, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381434341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381448781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[2]R flush_numbers[0..12]: [164, 155, 155, 162, 159, 163, 162, 160, 146, 162, 163, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381462711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[2]R generation[0..12]: [94, 90, 90, 93, 92, 94, 93, 92, 86, 93, 94, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381476231Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381490711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381504041Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381517271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381530151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:38.381543191Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"":"downstairs","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.38156004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.381572051Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"0","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.3815854Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"1","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.38159845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"client":"2","":"downstairs","session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.38161166Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.38162382Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"8e992ed0-307f-44e3-8ee4-58640ce6142a is now active with session: 4bac74bb-f9e7-439a-8d59-1e97ec36f399","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.38163937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.38165142Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576,"session_id":"4bac74bb-f9e7-439a-8d59-1e97ec36f399"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.38166338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} {"msg":"Activated sub_volume 8e992ed0-307f-44e3-8ee4-58640ce6142a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:38.3816745Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 111 len 40960 data: 5 5 6 6 7 9 9 10 10 9 02/30 Read block 355 len 24576 03/30 Read block 75 len 4096 04/30 Write block 141 len 16384 data: 6 6 6 4 05/30 Read block 474 len 20480 06/30 Read block 550 len 28672 07/30 Write block 109 len 24576 data: 8 6 6 6 7 7 08/30 Read block 415 len 4096 09/30 Read block 369 len 28672 10/30 Write block 26 len 4096 data: 12 11/30 Read block 489 len 40960 12/30 Read block 412 len 16384 13/30 Read block 377 len 32768 14/30 Write block 94 len 36864 data: 12 10 9 6 5 7 7 4 3 15/30 Read block 548 len 32768 16/30 Read block 347 len 4096 17/30 Write block 351 len 28672 data: 13 13 12 14 10 10 9 18/30 Read block 563 len 28672 19/30 Write block 66 len 20480 data: 7 7 6 7 7 20/30 Write block 129 len 20480 data: 8 9 13 12 11 21/30 Write block 334 len 36864 data: 7 7 7 9 10 8 7 6 6 22/30 Read block 422 len 20480 23/30 Read block 541 len 36864 24/30 Read block 522 len 16384 25/30 Read block 261 len 28672 26/30 Read block 101 len 36864 27/30 Write block 208 len 36864 data: 11 10 11 12 11 9 11 10 9 28/30 Read block 18 len 24576 29/30 Write block 21 len 16384 data: 10 9 12 11 30/30 Read block 136 len 28672 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:38.403550016Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 164 164 164 F F F 1 020-039 90 90 90 155 155 155 T T T 2 040-059 90 90 90 155 155 155 F F F 3 060-079 93 93 93 162 162 162 T T T 4 080-099 92 92 92 159 159 159 T T T 5 100-119 94 94 94 163 163 163 T T T 6 120-139 93 93 93 162 162 162 T T T 7 140-159 92 92 92 160 160 160 T T T 8 160-179 86 86 86 146 146 146 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 94 94 94 163 163 163 T T T 11 220-239 94 94 94 167 167 167 F F F 12 240-259 95 95 95 168 168 168 F F F 13 260-279 92 92 92 159 159 159 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 90 90 90 155 155 155 T T T 17 340-359 91 91 91 158 158 158 T T T 18 360-379 84 84 84 144 144 144 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 93 93 93 162 162 162 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 95 95 95 168 168 168 F F F 28 560-579 90 90 90 157 157 157 F F F 29 580-599 93 93 93 162 162 162 F F F Max gen: 95, Max flush: 168 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 164 164 164 F F F 1 020-039 96 96 96 169 169 169 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 96 96 96 169 169 169 F F F 4 080-099 96 96 96 169 169 169 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 96 96 96 169 169 169 F F F 7 140-159 96 96 96 169 169 169 F F F 8 160-179 86 86 86 146 146 146 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 94 94 94 167 167 167 F F F 12 240-259 95 95 95 168 168 168 F F F 13 260-279 92 92 92 159 159 159 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 F F F 16 320-339 96 96 96 169 169 169 F F F 17 340-359 96 96 96 169 169 169 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 93 93 93 162 162 162 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 94 94 94 163 163 163 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 95 95 95 168 168 168 F F F 28 560-579 90 90 90 157 157 157 F F F 29 580-599 93 93 93 162 162 162 F F F Max gen: 96, Max flush: 169 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.692264861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.735642406Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.736220232Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.736271512Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.736290602Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.736308681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} {"msg":"Crucible c084d474-315f-4049-b21d-5291a1fc09af has session id: cad07da5-1177-4a7a-8330-e659344026b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.736755068Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Upstairs opts: Upstairs UUID: c084d474-315f-4049-b21d-5291a1fc09af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.736802058Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Crucible stats registered with UUID: c084d474-315f-4049-b21d-5291a1fc09af","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.736830238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:42.736856277Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737337764Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737362863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} {"msg":"c084d474-315f-4049-b21d-5291a1fc09af active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737432443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737469623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"0","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737508622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"0","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737533662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"1","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737560432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"1","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737585432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"2","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737610082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"2","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737631792Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737656281Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737681671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64886)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737717271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65025)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737917539Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52675)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.737955659Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.738467505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"2","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.738495605Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"0","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.738513815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"1","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.73918636Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"1","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.739227309Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"2","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.739246949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"0","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.739810335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.739912964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[0]R flush_numbers[0..12]: [164, 169, 155, 169, 169, 169, 169, 169, 146, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.739983484Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[0]R generation[0..12]: [94, 96, 90, 96, 96, 96, 96, 96, 86, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740004364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740019094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[1]R flush_numbers[0..12]: [164, 169, 155, 169, 169, 169, 169, 169, 146, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740032854Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[1]R generation[0..12]: [94, 96, 90, 96, 96, 96, 96, 96, 86, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740046403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740059963Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[2]R flush_numbers[0..12]: [164, 169, 155, 169, 169, 169, 169, 169, 146, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740073423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[2]R generation[0..12]: [94, 96, 90, 96, 96, 96, 96, 96, 86, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740086593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740104553Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Max found gen is 97","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740118033Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Generation requested: 98 >= found:97","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740130883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740143483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740156193Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"":"downstairs","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:42.740169182Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740186892Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"0","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740200062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"1","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740212952Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"client":"2","":"downstairs","session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740226122Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740238142Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"c084d474-315f-4049-b21d-5291a1fc09af is now active with session: cad07da5-1177-4a7a-8330-e659344026b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740250612Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740262432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584,"session_id":"cad07da5-1177-4a7a-8330-e659344026b4"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740274382Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} {"msg":"Activated sub_volume c084d474-315f-4049-b21d-5291a1fc09af","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:42.740285422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 24576 data: 10 9 11 13 12 11 02/30 Write block 156 len 28672 data: 6 6 9 8 8 7 6 03/30 Read block 39 len 24576 04/30 Read block 357 len 16384 05/30 Write block 128 len 32768 data: 7 9 10 14 13 12 10 8 06/30 Write block 254 len 40960 data: 10 9 6 6 7 6 6 8 8 11 07/30 Write block 59 len 16384 data: 7 6 6 4 08/30 Write block 320 len 32768 data: 11 13 9 10 12 12 13 14 09/30 Read block 276 len 36864 10/30 Read block 123 len 16384 11/30 Read block 204 len 24576 12/30 Flush 13/30 Write block 556 len 24576 data: 14 15 13 11 11 11 14/30 Write block 72 len 28672 data: 8 8 7 9 9 8 9 15/30 Read block 81 len 8192 16/30 Write block 270 len 24576 data: 10 9 9 8 6 7 17/30 Flush 18/30 Write block 14 len 40960 data: 14 14 11 12 10 11 9 11 10 13 19/30 Write block 155 len 36864 data: 5 7 7 10 9 9 8 7 8 20/30 Write block 545 len 36864 data: 11 13 17 14 14 10 6 10 11 21/30 Read block 307 len 12288 22/30 Read block 89 len 24576 23/30 Read block 59 len 40960 24/30 Write block 434 len 8192 data: 11 10 25/30 Write block 315 len 8192 data: 7 8 26/30 Read block 272 len 20480 27/30 Write block 583 len 8192 data: 9 9 28/30 Write block 142 len 16384 data: 7 7 5 4 29/30 Write block 17 len 32768 data: 13 11 12 10 12 11 14 12 30/30 Read block 527 len 16384 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:42.760186003Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 164 164 164 T T T 1 020-039 96 96 96 169 169 169 T T T 2 040-059 98 98 98 170 170 170 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 96 96 96 169 169 169 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 98 98 98 170 170 170 T T T 8 160-179 98 98 98 170 170 170 T T T 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 94 94 94 167 167 167 F F F 12 240-259 98 98 98 170 170 170 F F F 13 260-279 98 98 98 171 171 171 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 85 85 85 145 145 145 T T T 16 320-339 98 98 98 170 170 170 F F F 17 340-359 96 96 96 169 169 169 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 93 93 93 162 162 162 T T T 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 98 98 98 170 170 170 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 98 98 98 171 171 171 T T T 28 560-579 98 98 98 171 171 171 F F F 29 580-599 93 93 93 162 162 162 T T T Max gen: 98, Max flush: 171 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 172 172 172 F F F 1 020-039 99 99 99 172 172 172 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 98 98 98 171 171 171 F F F 4 080-099 96 96 96 169 169 169 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 99 99 99 172 172 172 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 94 94 94 167 167 167 F F F 12 240-259 98 98 98 170 170 170 F F F 13 260-279 98 98 98 171 171 171 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 99 99 99 172 172 172 F F F 16 320-339 98 98 98 170 170 170 F F F 17 340-359 96 96 96 169 169 169 F F F 18 360-379 84 84 84 144 144 144 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 99 99 99 172 172 172 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 98 98 98 170 170 170 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 98 98 98 171 171 171 F F F 29 580-599 99 99 99 172 172 172 F F F Max gen: 99, Max flush: 172 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.019014311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:47.062392036Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.062752314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.094747714Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09534547Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.095385489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.095404809Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.095420619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} {"msg":"Crucible 4d2231ac-98c2-41f1-bd05-2db2f7d04d48 has session id: 0eadd853-7665-4dd0-aac2-f18d1f94d369","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096006505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Upstairs opts: Upstairs UUID: 4d2231ac-98c2-41f1-bd05-2db2f7d04d48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096045515Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Crucible stats registered with UUID: 4d2231ac-98c2-41f1-bd05-2db2f7d04d48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096065494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:47.096078924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096502721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096524301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} {"msg":"4d2231ac-98c2-41f1-bd05-2db2f7d04d48 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09660219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09662503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"0","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09666931Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"0","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09669797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"1","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096727559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"1","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096754399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"2","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096787429Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"2","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096813799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096844089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096870928Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"ds_connection connected from Ok(127.0.0.1:46322)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096933348Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"ds_connection connected from Ok(127.0.0.1:58278)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096965977Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"ds_connection connected from Ok(127.0.0.1:34031)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.096994387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.097589543Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"1","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.097618113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"2","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.097633812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"0","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.098207838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"1","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.098354157Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"0","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.098375137Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"2","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.098735234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.098940783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[0]R flush_numbers[0..12]: [172, 172, 170, 171, 169, 169, 170, 172, 172, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.098976742Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[0]R generation[0..12]: [99, 99, 98, 98, 96, 96, 98, 99, 99, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.098995652Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099011132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[1]R flush_numbers[0..12]: [172, 172, 170, 171, 169, 169, 170, 172, 172, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099025232Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[1]R generation[0..12]: [99, 99, 98, 98, 96, 96, 98, 99, 99, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099039422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099053542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[2]R flush_numbers[0..12]: [172, 172, 170, 171, 169, 169, 170, 172, 172, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099067372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[2]R generation[0..12]: [99, 99, 98, 98, 96, 96, 98, 99, 99, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099080942Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099094862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099107461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099120511Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099133521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099146321Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"":"downstairs","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099159551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099175561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"0","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099189331Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"1","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099202441Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"client":"2","":"downstairs","session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099215671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:47.099228061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"4d2231ac-98c2-41f1-bd05-2db2f7d04d48 is now active with session: 0eadd853-7665-4dd0-aac2-f18d1f94d369","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.099247871Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09925998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592,"session_id":"0eadd853-7665-4dd0-aac2-f18d1f94d369"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09927271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} {"msg":"Activated sub_volume 4d2231ac-98c2-41f1-bd05-2db2f7d04d48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:47.09928401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 38 len 4096 data: 14 03/30 Read block 120 len 12288 04/30 Read block 137 len 40960 05/30 Write block 35 len 40960 data: 11 13 12 15 14 10 9 6 7 6 06/30 Write block 283 len 20480 data: 4 7 10 8 10 07/30 Read block 462 len 20480 08/30 Read block 297 len 12288 09/30 Read block 27 len 28672 10/30 Write block 71 len 16384 data: 7 9 9 8 11/30 Read block 531 len 24576 12/30 Write block 375 len 24576 data: 8 7 6 9 7 7 13/30 Read block 211 len 12288 14/30 Write block 338 len 40960 data: 11 9 8 7 7 7 7 7 7 10 15/30 Flush 16/30 Write block 93 len 8192 data: 12 13 17/30 Read block 341 len 32768 18/30 Read block 223 len 40960 19/30 Flush 20/30 Flush 21/30 Write block 481 len 28672 data: 5 8 10 10 10 12 11 22/30 Flush 23/30 Flush 24/30 Write block 309 len 16384 data: 5 5 6 7 25/30 Write block 334 len 32768 data: 8 8 8 10 12 10 9 8 26/30 Write block 48 len 8192 data: 5 6 27/30 Read block 519 len 40960 28/30 Write block 489 len 40960 data: 11 10 12 14 13 12 11 7 9 10 29/30 Write block 564 len 36864 data: 7 8 6 5 7 8 9 7 7 30/30 Write block 244 len 28672 data: 6 5 5 6 10 10 9 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:47.119828776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 172 172 172 F F F 1 020-039 100 100 100 174 174 174 F F F 2 040-059 100 100 100 174 174 174 T T T 3 060-079 100 100 100 174 174 174 F F F 4 080-099 100 100 100 175 175 175 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 99 99 99 172 172 172 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 94 94 94 167 167 167 F F F 12 240-259 98 98 98 170 170 170 T T T 13 260-279 98 98 98 171 171 171 F F F 14 280-299 100 100 100 174 174 174 F F F 15 300-319 99 99 99 172 172 172 T T T 16 320-339 100 100 100 174 174 174 T T T 17 340-359 100 100 100 174 174 174 T T T 18 360-379 100 100 100 174 174 174 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 99 99 99 172 172 172 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 100 100 100 177 177 177 T T T 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 98 98 98 171 171 171 T T T 29 580-599 99 99 99 172 172 172 F F F Max gen: 100, Max flush: 177 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 172 172 172 F F F 1 020-039 100 100 100 174 174 174 F F F 2 040-059 101 101 101 178 178 178 F F F 3 060-079 100 100 100 174 174 174 F F F 4 080-099 100 100 100 175 175 175 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 99 99 99 172 172 172 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 94 94 94 167 167 167 F F F 12 240-259 101 101 101 178 178 178 F F F 13 260-279 98 98 98 171 171 171 F F F 14 280-299 100 100 100 174 174 174 F F F 15 300-319 101 101 101 178 178 178 F F F 16 320-339 101 101 101 178 178 178 F F F 17 340-359 101 101 101 178 178 178 F F F 18 360-379 100 100 100 174 174 174 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 99 99 99 172 172 172 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 94 94 94 165 165 165 F F F 24 480-499 101 101 101 178 178 178 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 101 101 101 178 178 178 F F F 29 580-599 99 99 99 172 172 172 F F F Max gen: 101, Max flush: 178 Begin loop 51 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.397987021Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.441552254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.44220373Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.442257589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.442276899Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.442295119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} {"msg":"Crucible 743cd9fe-54d7-4a7f-a6e0-70fee45c701d has session id: f2c2cf09-728a-4448-bda5-e3bd9498a46d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.442849594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Upstairs opts: Upstairs UUID: 743cd9fe-54d7-4a7f-a6e0-70fee45c701d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.442895594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Crucible stats registered with UUID: 743cd9fe-54d7-4a7f-a6e0-70fee45c701d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.442915434Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:51.442931174Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.44345957Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.44348069Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} {"msg":"743cd9fe-54d7-4a7f-a6e0-70fee45c701d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443569669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443596499Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"0","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443625859Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"0","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443640029Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"1","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443654219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"1","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443667079Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"2","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443684928Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"2","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443708018Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443731428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443754638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55609)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443779388Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56229)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.443951006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63176)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.444055636Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.444506872Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"1","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.44473928Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"0","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.445072828Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"2","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.445125528Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"1","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.445320586Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"0","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.445711683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.445812752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.445833342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"2","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[0]R flush_numbers[0..12]: [172, 174, 178, 174, 175, 169, 170, 172, 172, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446371608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[0]R generation[0..12]: [99, 100, 101, 100, 100, 96, 98, 99, 99, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446399678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446415718Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[1]R flush_numbers[0..12]: [172, 174, 178, 174, 175, 169, 170, 172, 172, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446430158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[1]R generation[0..12]: [99, 100, 101, 100, 100, 96, 98, 99, 99, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446443848Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446458307Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[2]R flush_numbers[0..12]: [172, 174, 178, 174, 175, 169, 170, 172, 172, 162, 169, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446472147Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[2]R generation[0..12]: [99, 100, 101, 100, 100, 96, 98, 99, 99, 93, 96, 94]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446485467Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446502767Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446526087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446557137Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446581067Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446604866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"":"downstairs","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446624056Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446636526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"0","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446649606Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"1","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446662586Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"client":"2","":"downstairs","session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446675546Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446687676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"743cd9fe-54d7-4a7f-a6e0-70fee45c701d is now active with session: f2c2cf09-728a-4448-bda5-e3bd9498a46d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446701446Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446713516Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600,"session_id":"f2c2cf09-728a-4448-bda5-e3bd9498a46d"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:51.446725676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} {"msg":"Activated sub_volume 743cd9fe-54d7-4a7f-a6e0-70fee45c701d","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:04:51.446736785Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 490 len 12288 data: 11 13 15 02/30 Write block 478 len 12288 data: 4 6 5 03/30 Read block 540 len 8192 04/30 Write block 426 len 16384 data: 7 7 5 7 05/30 Read block 148 len 28672 06/30 Read block 531 len 16384 07/30 Read block 444 len 40960 08/30 Write block 589 len 8192 data: 8 8 09/30 Read block 305 len 8192 10/30 Write block 357 len 36864 data: 10 8 10 8 6 5 5 6 6 11/30 Read block 167 len 24576 12/30 Read block 353 len 28672 13/30 Write block 429 len 12288 data: 8 8 12 14/30 Read block 363 len 20480 15/30 Read block 451 len 16384 16/30 Write block 590 len 16384 data: 9 7 8 7 17/30 Write block 220 len 8192 data: 5 6 18/30 Read block 319 len 36864 19/30 Write block 482 len 36864 data: 9 11 11 11 13 12 10 12 12 20/30 Write block 147 len 36864 data: 4 6 7 9 10 11 9 7 6 21/30 Write block 91 len 28672 data: 10 12 13 14 11 10 7 22/30 Flush 23/30 Write block 56 len 8192 data: 10 8 24/30 Write block 492 len 12288 data: 16 14 13 25/30 Read block 400 len 4096 26/30 Write block 580 len 8192 data: 10 7 27/30 Write block 326 len 28672 data: 14 15 10 7 8 7 7 28/30 Read block 402 len 8192 29/30 Write block 301 len 24576 data: 8 7 6 7 7 6 30/30 Read block 216 len 4096 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:51.465909632Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 172 172 172 F F F 1 020-039 100 100 100 174 174 174 F F F 2 040-059 101 101 101 178 178 178 T T T 3 060-079 100 100 100 174 174 174 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 102 102 102 179 179 179 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 102 102 102 179 179 179 F F F 12 240-259 101 101 101 178 178 178 F F F 13 260-279 98 98 98 171 171 171 F F F 14 280-299 100 100 100 174 174 174 F F F 15 300-319 101 101 101 178 178 178 T T T 16 320-339 101 101 101 178 178 178 T T T 17 340-359 102 102 102 179 179 179 F F F 18 360-379 102 102 102 179 179 179 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 102 102 102 179 179 179 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 102 102 102 179 179 179 F F F 24 480-499 102 102 102 179 179 179 T T T 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 101 101 101 178 178 178 F F F 29 580-599 102 102 102 179 179 179 T T T Max gen: 102, Max flush: 179 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 172 172 172 F F F 1 020-039 100 100 100 174 174 174 F F F 2 040-059 103 103 103 180 180 180 F F F 3 060-079 100 100 100 174 174 174 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 98 98 98 170 170 170 F F F 7 140-159 102 102 102 179 179 179 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 102 102 102 179 179 179 F F F 12 240-259 101 101 101 178 178 178 F F F 13 260-279 98 98 98 171 171 171 F F F 14 280-299 100 100 100 174 174 174 F F F 15 300-319 103 103 103 180 180 180 F F F 16 320-339 103 103 103 180 180 180 F F F 17 340-359 102 102 102 179 179 179 F F F 18 360-379 102 102 102 179 179 179 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 102 102 102 179 179 179 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 102 102 102 179 179 179 F F F 24 480-499 103 103 103 180 180 180 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 90 90 90 157 157 157 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 101 101 101 178 178 178 F F F 29 580-599 103 103 103 180 180 180 F F F Max gen: 103, Max flush: 180 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.7021365Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.746949755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.74761665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.747660829Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.747676419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.747690849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} {"msg":"Crucible b8ddff8e-1021-42e2-a837-088bc41c938c has session id: f0b9820a-8eea-4774-a484-bf6e067f91b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748234095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Upstairs opts: Upstairs UUID: b8ddff8e-1021-42e2-a837-088bc41c938c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748302254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Crucible stats registered with UUID: b8ddff8e-1021-42e2-a837-088bc41c938c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748330054Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:55.748344524Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748712961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748736051Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} {"msg":"b8ddff8e-1021-42e2-a837-088bc41c938c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748784181Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748808211Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"0","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.74883666Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"0","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"1","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.74891533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"1","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.74893368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"2","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.74894804Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"2","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748961129Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.748983169Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.749012479Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54949)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.749039119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63239)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.749067149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:55517)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.749094039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.749781503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"1","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.749809933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"2","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.749825333Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"0","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.750420578Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"1","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.750449018Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"2","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.750464268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"0","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751058464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751085664Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[0]R flush_numbers[0..12]: [172, 174, 180, 174, 179, 169, 170, 179, 172, 162, 169, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751115863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[0]R generation[0..12]: [99, 100, 103, 100, 102, 96, 98, 102, 99, 93, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751132763Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751151463Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[1]R flush_numbers[0..12]: [172, 174, 180, 174, 179, 169, 170, 179, 172, 162, 169, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751166713Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[1]R generation[0..12]: [99, 100, 103, 100, 102, 96, 98, 102, 99, 93, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751181273Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751195233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[2]R flush_numbers[0..12]: [172, 174, 180, 174, 179, 169, 170, 179, 172, 162, 169, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751208823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[2]R generation[0..12]: [99, 100, 103, 100, 102, 96, 98, 102, 99, 93, 96, 102]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751222272Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751235943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751248862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751266952Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751283862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751296892Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"":"downstairs","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751309862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751321752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"0","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751334912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"1","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751347891Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"client":"2","":"downstairs","session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751363462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751375801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFXWait for a query_work_queue command to finish before sending IO ","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"b8ddff8e-1021-42e2-a837-088bc41c938c is now active with session: f0b9820a-8eea-4774-a484-bf6e067f91b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751397731Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751409461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608,"session_id":"f0b9820a-8eea-4774-a484-bf6e067f91b4"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751421541Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} {"msg":"Activated sub_volume b8ddff8e-1021-42e2-a837-088bc41c938c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:55.751432591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 526 len 40960 data: 10 8 6 6 5 6 5 6 6 5 02/30 Write block 220 len 12288 data: 6 7 6 03/30 Read block 177 len 28672 04/30 Flush 05/30 Write block 10 len 28672 data: 12 14 11 10 15 15 12 06/30 Read block 52 len 16384 07/30 Write block 47 len 36864 data: 6 6 7 5 6 5 6 7 10 08/30 Read block 473 len 40960 09/30 Flush 10/30 Write block 41 len 12288 data: 10 7 8 11/30 Read block 81 len 20480 12/30 Read block 383 len 24576 13/30 Write block 12 len 28672 data: 12 11 16 16 13 14 12 14/30 Write block 19 len 40960 data: 13 11 13 12 15 13 11 13 10 8 15/30 Read block 582 len 36864 16/30 Read block 427 len 12288 17/30 Read block 126 len 16384 18/30 Write block 343 len 20480 data: 8 8 8 8 11 19/30 Read block 214 len 12288 20/30 Write block 5 len 12288 data: 8 9 11 21/30 Read block 580 len 28672 22/30 Write block 143 len 28672 data: 8 6 5 5 5 7 8 23/30 Read block 38 len 4096 24/30 Write block 52 len 40960 data: 6 7 8 11 11 9 8 8 7 7 25/30 Read block 118 len 32768 26/30 Flush 27/30 Write block 6 len 24576 data: 10 12 11 13 13 15 28/30 Write block 138 len 32768 data: 6 6 9 7 8 9 7 6 29/30 Read block 397 len 20480 30/30 Read block 276 len 36864 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:55.773514076Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 183 183 183 T T T 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 104 104 104 183 183 183 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 98 98 98 170 170 170 T T T 7 140-159 104 104 104 183 183 183 T T T 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 104 104 104 181 181 181 F F F 12 240-259 101 101 101 178 178 178 F F F 13 260-279 98 98 98 171 171 171 F F F 14 280-299 100 100 100 174 174 174 F F F 15 300-319 103 103 103 180 180 180 F F F 16 320-339 103 103 103 180 180 180 F F F 17 340-359 104 104 104 183 183 183 F F F 18 360-379 102 102 102 179 179 179 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 102 102 102 179 179 179 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 102 102 102 179 179 179 F F F 24 480-499 103 103 103 180 180 180 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 104 104 104 181 181 181 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 101 101 101 178 178 178 F F F 29 580-599 103 103 103 180 180 180 F F F Max gen: 104, Max flush: 183 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 184 184 184 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 104 104 104 183 183 183 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 105 105 105 184 184 184 F F F 7 140-159 105 105 105 184 184 184 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 F F F 11 220-239 104 104 104 181 181 181 F F F 12 240-259 101 101 101 178 178 178 F F F 13 260-279 98 98 98 171 171 171 F F F 14 280-299 100 100 100 174 174 174 F F F 15 300-319 103 103 103 180 180 180 F F F 16 320-339 103 103 103 180 180 180 F F F 17 340-359 104 104 104 183 183 183 F F F 18 360-379 102 102 102 179 179 179 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 94 94 94 167 167 167 F F F 21 420-439 102 102 102 179 179 179 F F F 22 440-459 94 94 94 167 167 167 F F F 23 460-479 102 102 102 179 179 179 F F F 24 480-499 103 103 103 180 180 180 F F F 25 500-519 94 94 94 167 167 167 F F F 26 520-539 104 104 104 181 181 181 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 101 101 101 178 178 178 F F F 29 580-599 103 103 103 180 180 180 F F F Max gen: 105, Max flush: 184 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.002744715Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.047950776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.048586502Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.048643441Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.048667041Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.048691521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} {"msg":"Crucible 29cdb032-edab-4f5e-873c-5cd843cb31f4 has session id: eda0764a-3fb5-47ef-8ab8-80971a74aed1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049130918Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Upstairs opts: Upstairs UUID: 29cdb032-edab-4f5e-873c-5cd843cb31f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049169187Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Crucible stats registered with UUID: 29cdb032-edab-4f5e-873c-5cd843cb31f4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049185757Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:00.049198807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049631594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049655254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} {"msg":"29cdb032-edab-4f5e-873c-5cd843cb31f4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049712623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049739173Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"0","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049770113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"0","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049789603Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"1","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049808503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"1","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049830973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"2","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049855312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"2","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049877692Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049903082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049927042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40927)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049951932Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46852)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.049977751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45939)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.050004991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.050744036Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"2","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.050785295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"0","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.050814135Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"1","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.051381911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"0","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.05151277Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"1","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.05154232Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"2","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.051943237Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052049786Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[0]R flush_numbers[0..12]: [184, 183, 183, 183, 179, 169, 184, 184, 172, 162, 169, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052172795Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[0]R generation[0..12]: [105, 104, 104, 104, 102, 96, 105, 105, 99, 93, 96, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052202155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052218415Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[1]R flush_numbers[0..12]: [184, 183, 183, 183, 179, 169, 184, 184, 172, 162, 169, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052232784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[1]R generation[0..12]: [105, 104, 104, 104, 102, 96, 105, 105, 99, 93, 96, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052246914Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052260794Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[2]R flush_numbers[0..12]: [184, 183, 183, 183, 179, 169, 184, 184, 172, 162, 169, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052274404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[2]R generation[0..12]: [105, 104, 104, 104, 102, 96, 105, 105, 99, 93, 96, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052287784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052301634Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052314244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052327144Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052350104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052362854Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"":"downstairs","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052376034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052388093Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"0","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052401503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"1","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052414593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"client":"2","":"downstairs","session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052428453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052441013Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"29cdb032-edab-4f5e-873c-5cd843cb31f4 is now active with session: eda0764a-3fb5-47ef-8ab8-80971a74aed1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052453943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:05:00.052465863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616,"session_id":"eda0764a-3fb5-47ef-8ab8-80971a74aed1"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052478233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} {"msg":"Activated sub_volume 29cdb032-edab-4f5e-873c-5cd843cb31f4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:00.052489282Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 4096 data: 8 02/30 Read block 217 len 4096 03/30 Write block 271 len 20480 data: 10 10 9 7 8 04/30 Read block 78 len 32768 05/30 Write block 134 len 20480 data: 11 9 8 7 7 06/30 Write block 407 len 28672 data: 10 9 10 9 9 8 8 07/30 Flush 08/30 Write block 286 len 40960 data: 9 11 12 11 12 13 14 15 10 7 09/30 Read block 326 len 28672 10/30 Write block 252 len 20480 data: 6 7 11 10 7 11/30 Write block 428 len 36864 data: 6 9 9 13 10 11 12 11 8 12/30 Write block 585 len 20480 data: 9 9 9 9 9 13/30 Read block 125 len 40960 14/30 Write block 73 len 4096 data: 10 15/30 Read block 343 len 36864 16/30 Read block 59 len 40960 17/30 Flush 18/30 Write block 342 len 24576 data: 8 9 9 9 9 12 19/30 Write block 241 len 4096 data: 7 20/30 Flush 21/30 Write block 316 len 12288 data: 9 8 7 22/30 Write block 200 len 28672 data: 7 8 7 6 6 7 9 23/30 Write block 346 len 28672 data: 10 13 8 10 10 14 14 24/30 Read block 75 len 24576 25/30 Read block 227 len 4096 26/30 Read block 591 len 8192 27/30 Read block 317 len 20480 28/30 Read block 527 len 40960 29/30 Write block 459 len 16384 data: 5 6 7 8 30/30 Write block 2 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:00.072075326Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 184 184 184 T T T 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 106 106 106 185 185 185 F F F 7 140-159 105 105 105 184 184 184 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 96 96 96 169 169 169 T T T 11 220-239 104 104 104 181 181 181 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 106 106 106 186 186 186 F F F 15 300-319 103 103 103 180 180 180 T T T 16 320-339 103 103 103 180 180 180 F F F 17 340-359 106 106 106 187 187 187 T T T 18 360-379 102 102 102 179 179 179 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 106 106 106 185 185 185 F F F 21 420-439 106 106 106 186 186 186 F F F 22 440-459 94 94 94 167 167 167 T T T 23 460-479 102 102 102 179 179 179 T T T 24 480-499 103 103 103 180 180 180 F F F 25 500-519 106 106 106 185 185 185 F F F 26 520-539 104 104 104 181 181 181 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 101 101 101 178 178 178 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 106, Max flush: 187 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 188 188 188 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 106 106 106 185 185 185 F F F 7 140-159 105 105 105 184 184 184 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 F F F 10 200-219 107 107 107 188 188 188 F F F 11 220-239 104 104 104 181 181 181 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 106 106 106 186 186 186 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 103 103 103 180 180 180 F F F 17 340-359 107 107 107 188 188 188 F F F 18 360-379 102 102 102 179 179 179 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 106 106 106 185 185 185 F F F 21 420-439 106 106 106 186 186 186 F F F 22 440-459 107 107 107 188 188 188 F F F 23 460-479 107 107 107 188 188 188 F F F 24 480-499 103 103 103 180 180 180 F F F 25 500-519 106 106 106 185 185 185 F F F 26 520-539 104 104 104 181 181 181 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 101 101 101 178 178 178 F F F 29 580-599 106 106 106 186 186 186 F F F Max gen: 107, Max flush: 188 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.315604079Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.360263905Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36087989Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.3609432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36096236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36097776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} {"msg":"Crucible bb6167c4-fd00-4c3c-a8de-ac0c9fed405d has session id: 17f0e0ac-a6e1-4628-b732-b47d463fb521","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.361456436Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Upstairs opts: Upstairs UUID: bb6167c4-fd00-4c3c-a8de-ac0c9fed405d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.361495676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Crucible stats registered with UUID: bb6167c4-fd00-4c3c-a8de-ac0c9fed405d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.361527466Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:04.361628235Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362119531Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362146311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} {"msg":"bb6167c4-fd00-4c3c-a8de-ac0c9fed405d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36221937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36223735Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"0","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.3622654Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"0","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36227929Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"1","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36229366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"1","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36230711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"2","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362321029Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"2","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36233556Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362359939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362386089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"ds_connection connected from Ok(127.0.0.1:57091)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362590467Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"ds_connection connected from Ok(127.0.0.1:63016)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362629887Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"ds_connection connected from Ok(127.0.0.1:47885)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.362662797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.363145683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"2","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.363175783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"0","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.363192913Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"1","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.363827718Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"0","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.363856598Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"2","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.363873388Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"1","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364397884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364512903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[0]R flush_numbers[0..12]: [188, 183, 183, 186, 179, 169, 185, 184, 172, 162, 188, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364574753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[0]R generation[0..12]: [107, 104, 104, 106, 102, 96, 106, 105, 99, 93, 107, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364594312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364609823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[1]R flush_numbers[0..12]: [188, 183, 183, 186, 179, 169, 185, 184, 172, 162, 188, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364624322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[1]R generation[0..12]: [107, 104, 104, 106, 102, 96, 106, 105, 99, 93, 107, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364638072Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364655892Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[2]R flush_numbers[0..12]: [188, 183, 183, 186, 179, 169, 185, 184, 172, 162, 188, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364670092Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[2]R generation[0..12]: [107, 104, 104, 106, 102, 96, 106, 105, 99, 93, 107, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364683912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364698472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364712062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364726682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364746671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364759591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"":"downstairs","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364772701Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364784621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"0","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364798071Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"1","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364811251Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"client":"2","":"downstairs","session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364824431Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364836551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"bb6167c4-fd00-4c3c-a8de-ac0c9fed405d is now active with session: 17f0e0ac-a6e1-4628-b732-b47d463fb521","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:05:04.364849391Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.364864781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624,"session_id":"17f0e0ac-a6e1-4628-b732-b47d463fb521"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36488414Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} {"msg":"Activated sub_volume bb6167c4-fd00-4c3c-a8de-ac0c9fed405d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:04.36489592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 120 len 36864 data: 10 8 8 9 11 10 8 9 8 04/30 Read block 372 len 12288 05/30 Read block 23 len 28672 06/30 Flush 07/30 Write block 438 len 12288 data: 10 11 8 08/30 Flush 09/30 Write block 281 len 28672 data: 7 6 5 8 11 10 12 10/30 Read block 391 len 16384 11/30 Write block 467 len 40960 data: 7 7 7 9 7 6 6 7 6 5 12/30 Write block 513 len 24576 data: 8 3 4 4 6 8 13/30 Flush 14/30 Write block 204 len 4096 data: 7 15/30 Write block 578 len 20480 data: 9 9 11 8 9 16/30 Write block 288 len 16384 data: 13 12 13 14 17/30 Read block 192 len 32768 18/30 Write block 374 len 20480 data: 9 9 8 7 10 19/30 Flush 20/30 Write block 351 len 32768 data: 15 15 13 15 11 11 11 9 21/30 Write block 136 len 16384 data: 9 8 8 7 22/30 Read block 415 len 28672 23/30 Write block 538 len 32768 data: 5 6 6 9 9 8 10 12 24/30 Read block 12 len 4096 25/30 Write block 383 len 28672 data: 5 7 8 10 7 6 7 26/30 Read block 154 len 20480 27/30 Write block 193 len 32768 data: 6 6 8 8 7 5 6 8 28/30 Read block 331 len 24576 29/30 Write block 128 len 40960 data: 9 10 11 15 14 13 12 10 10 9 30/30 Write block 515 len 32768 data: 5 5 7 9 11 11 14 14 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:04.38495984Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 188 188 188 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 108 108 108 191 191 191 T T T 7 140-159 105 105 105 184 184 184 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 93 93 93 162 162 162 T T T 10 200-219 108 108 108 194 194 194 T T T 11 220-239 104 104 104 181 181 181 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 103 103 103 180 180 180 F F F 17 340-359 107 107 107 188 188 188 T T T 18 360-379 108 108 108 194 194 194 F F F 19 380-399 100 100 100 174 174 174 T T T 20 400-419 106 106 106 185 185 185 F F F 21 420-439 108 108 108 192 192 192 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 108 108 108 193 193 193 F F F 24 480-499 103 103 103 180 180 180 F F F 25 500-519 108 108 108 193 193 193 T T T 26 520-539 104 104 104 181 181 181 T T T 27 540-559 99 99 99 172 172 172 T T T 28 560-579 108 108 108 194 194 194 F F F 29 580-599 108 108 108 194 194 194 F F F Max gen: 108, Max flush: 194 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 188 188 188 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 106 106 106 186 186 186 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 109 109 109 195 195 195 F F F 7 140-159 105 105 105 184 184 184 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 109 109 109 195 195 195 F F F 11 220-239 104 104 104 181 181 181 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 103 103 103 180 180 180 F F F 17 340-359 109 109 109 195 195 195 F F F 18 360-379 108 108 108 194 194 194 F F F 19 380-399 109 109 109 195 195 195 F F F 20 400-419 106 106 106 185 185 185 F F F 21 420-439 108 108 108 192 192 192 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 108 108 108 193 193 193 F F F 24 480-499 103 103 103 180 180 180 F F F 25 500-519 109 109 109 195 195 195 F F F 26 520-539 109 109 109 195 195 195 F F F 27 540-559 109 109 109 195 195 195 F F F 28 560-579 108 108 108 194 194 194 F F F 29 580-599 108 108 108 194 194 194 F F F Max gen: 109, Max flush: 195 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.634844996Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:08.677278568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.677634286Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.708402365Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.708946331Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.708981371Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.709009661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.709026571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} {"msg":"Crucible c22cffb9-204e-4f98-aa99-bf598d7fc5f6 has session id: e5d8456c-d114-4971-8f7a-cd34d05afa57","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.709585747Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Upstairs opts: Upstairs UUID: c22cffb9-204e-4f98-aa99-bf598d7fc5f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.709626356Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Crucible stats registered with UUID: c22cffb9-204e-4f98-aa99-bf598d7fc5f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.709649856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:08.709670566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710068633Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710094893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} {"msg":"c22cffb9-204e-4f98-aa99-bf598d7fc5f6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710155062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710173192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"0","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710207162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"0","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710237972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"1","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710267431Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"1","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710295581Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"2","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710324541Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"2","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710351711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.71037941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.71041705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"ds_connection connected from Ok(127.0.0.1:51873)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.7104453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"ds_connection connected from Ok(127.0.0.1:47464)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710508349Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"ds_connection connected from Ok(127.0.0.1:55209)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.710535299Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.711081295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"1","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.711112735Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"2","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.711397993Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"0","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.71185153Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"1","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.711886259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"2","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.711969819Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"0","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712498045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712534005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[0]R flush_numbers[0..12]: [188, 183, 183, 186, 179, 169, 195, 184, 172, 195, 195, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712552704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[0]R generation[0..12]: [107, 104, 104, 106, 102, 96, 109, 105, 99, 109, 109, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712567504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712581684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[1]R flush_numbers[0..12]: [188, 183, 183, 186, 179, 169, 195, 184, 172, 195, 195, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712595364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[1]R generation[0..12]: [107, 104, 104, 106, 102, 96, 109, 105, 99, 109, 109, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712609034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712622694Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[2]R flush_numbers[0..12]: [188, 183, 183, 186, 179, 169, 195, 184, 172, 195, 195, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712636394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[2]R generation[0..12]: [107, 104, 104, 106, 102, 96, 109, 105, 99, 109, 109, 104]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712649813Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712663473Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712676183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712693423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712706893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712719723Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"":"downstairs","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712739523Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712761583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"0","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712782403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"1","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:08.712806802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"client":"2","":"downstairs","session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712833012Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712857382Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"c22cffb9-204e-4f98-aa99-bf598d7fc5f6 is now active with session: e5d8456c-d114-4971-8f7a-cd34d05afa57","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712873162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712885432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632,"session_id":"e5d8456c-d114-4971-8f7a-cd34d05afa57"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712904532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} {"msg":"Activated sub_volume c22cffb9-204e-4f98-aa99-bf598d7fc5f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:08.712925311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 359 len 4096 data: 11 02/30 Read block 234 len 24576 03/30 Read block 423 len 8192 04/30 Read block 170 len 36864 05/30 Read block 265 len 4096 06/30 Read block 204 len 28672 07/30 Read block 68 len 32768 08/30 Read block 120 len 32768 09/30 Flush 10/30 Write block 217 len 36864 data: 9 7 7 7 8 7 7 6 8 11/30 Write block 479 len 32768 data: 7 6 6 10 12 12 12 14 12/30 Write block 402 len 8192 data: 9 12 13/30 Read block 175 len 40960 14/30 Write block 329 len 4096 data: 8 15/30 Read block 253 len 20480 16/30 Read block 37 len 12288 17/30 Write block 65 len 32768 data: 6 8 8 7 8 8 8 10 18/30 Read block 52 len 16384 19/30 Read block 493 len 32768 20/30 Read block 66 len 28672 21/30 Read block 413 len 8192 22/30 Read block 568 len 40960 23/30 Write block 1 len 28672 data: 6 7 7 7 9 11 13 24/30 Read block 511 len 20480 25/30 Write block 539 len 8192 data: 7 7 26/30 Read block 175 len 28672 27/30 Read block 41 len 20480 28/30 Read block 62 len 24576 29/30 Read block 186 len 20480 30/30 Read block 328 len 20480 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 9 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 20 20 21 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:08.733631956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 188 188 188 T T T 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 106 106 106 186 186 186 T T T 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 109 109 109 195 195 195 F F F 7 140-159 105 105 105 184 184 184 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 109 109 109 195 195 195 T T T 11 220-239 104 104 104 181 181 181 T T T 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 103 103 103 180 180 180 T T T 17 340-359 110 110 110 196 196 196 F F F 18 360-379 108 108 108 194 194 194 F F F 19 380-399 109 109 109 195 195 195 F F F 20 400-419 106 106 106 185 185 185 T T T 21 420-439 108 108 108 192 192 192 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 108 108 108 193 193 193 T T T 24 480-499 103 103 103 180 180 180 T T T 25 500-519 109 109 109 195 195 195 F F F 26 520-539 109 109 109 195 195 195 T T T 27 540-559 109 109 109 195 195 195 T T T 28 560-579 108 108 108 194 194 194 F F F 29 580-599 108 108 108 194 194 194 F F F Max gen: 110, Max flush: 196 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 197 197 197 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 104 104 104 183 183 183 F F F 3 060-079 111 111 111 197 197 197 F F F 4 080-099 102 102 102 179 179 179 F F F 5 100-119 96 96 96 169 169 169 F F F 6 120-139 109 109 109 195 195 195 F F F 7 140-159 105 105 105 184 184 184 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 111 111 111 197 197 197 F F F 11 220-239 111 111 111 197 197 197 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 111 111 111 197 197 197 F F F 17 340-359 110 110 110 196 196 196 F F F 18 360-379 108 108 108 194 194 194 F F F 19 380-399 109 109 109 195 195 195 F F F 20 400-419 111 111 111 197 197 197 F F F 21 420-439 108 108 108 192 192 192 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 111 111 111 197 197 197 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 109 109 109 195 195 195 F F F 26 520-539 111 111 111 197 197 197 F F F 27 540-559 111 111 111 197 197 197 F F F 28 560-579 108 108 108 194 194 194 F F F 29 580-599 108 108 108 194 194 194 F F F Max gen: 111, Max flush: 197 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.026749509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:13.069980645Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.070368313Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.101826057Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.102384443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.102426393Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.102443122Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.102465042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} {"msg":"Crucible 793fd266-a1e2-4c1d-9423-2d654c4b1ec8 has session id: bb627218-6b7d-4fe0-aff4-9dcaa2794dfc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.102949609Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Upstairs opts: Upstairs UUID: 793fd266-a1e2-4c1d-9423-2d654c4b1ec8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.102989858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Crucible stats registered with UUID: 793fd266-a1e2-4c1d-9423-2d654c4b1ec8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103008798Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:13.103025698Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103442255Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103475145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} {"msg":"793fd266-a1e2-4c1d-9423-2d654c4b1ec8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103531524Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103558624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"0","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103589224Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"0","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103605584Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"1","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103631314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"1","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103655933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"2","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103681053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"2","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103706233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103737423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103762073Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"ds_connection connected from Ok(127.0.0.1:51690)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103786752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34890)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103814542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45790)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.103988701Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.104490657Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"1","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.104527697Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"2","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.104721296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"0","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105187542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"2","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105218032Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"1","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105285711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"0","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105794888Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105817497Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[0]R flush_numbers[0..12]: [197, 183, 183, 197, 179, 169, 195, 184, 172, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105873307Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[0]R generation[0..12]: [111, 104, 104, 111, 102, 96, 109, 105, 99, 109, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105897307Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105912347Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[1]R flush_numbers[0..12]: [197, 183, 183, 197, 179, 169, 195, 184, 172, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105926176Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[1]R generation[0..12]: [111, 104, 104, 111, 102, 96, 109, 105, 99, 109, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105939756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105953676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[2]R flush_numbers[0..12]: [197, 183, 183, 197, 179, 169, 195, 184, 172, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105967426Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[2]R generation[0..12]: [111, 104, 104, 111, 102, 96, 109, 105, 99, 109, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105980936Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.105994816Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106014276Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106034346Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106050555Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106063555Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"":"downstairs","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106076545Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106088515Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"0","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106101735Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"1","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106116665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"client":"2","":"downstairs","session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106129895Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106149525Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"793fd266-a1e2-4c1d-9423-2d654c4b1ec8 is now active with session: bb627218-6b7d-4fe0-aff4-9dcaa2794dfc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106162375Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106174145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640,"session_id":"bb627218-6b7d-4fe0-aff4-9dcaa2794dfc"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106186174Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} {"msg":"Activated sub_volume 793fd266-a1e2-4c1d-9423-2d654c4b1ec8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:13.106197314Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 330 len 4096 data: 9 03/30 Read block 54 len 36864 04/30 Read block 578 len 8192 05/30 Write block 576 len 28672 data: 9 10 10 10 12 9 10 06/30 Read block 337 len 36864 07/30 Read block 580 len 20480 08/30 Write block 367 len 16384 data: 7 7 6 7 09/30 Read block 461 len 12288 10/30 Write block 336 len 20480 data: 9 11 13 11 10 11/30 Write block 46 len 12288 data: 5 7 7 12/30 Write block 122 len 40960 data: 9 10 12 11 9 10 10 11 12 16 13/30 Read block 356 len 4096 14/30 Read block 573 len 8192 15/30 Read block 24 len 12288 16/30 Read block 443 len 32768 17/30 Read block 45 len 12288 18/30 Flush 19/30 Write block 477 len 12288 data: 4 5 8 20/30 Write block 74 len 24576 data: 9 10 10 9 10 9 21/30 Write block 549 len 36864 data: 15 11 7 11 12 10 13 15 16 22/30 Write block 108 len 28672 data: 7 9 7 7 7 8 8 23/30 Write block 97 len 40960 data: 8 6 8 8 5 4 6 7 7 7 24/30 Write block 554 len 32768 data: 11 14 16 17 14 12 12 12 25/30 Read block 320 len 16384 26/30 Read block 338 len 36864 27/30 Read block 199 len 20480 28/30 Read block 444 len 40960 29/30 Write block 135 len 32768 data: 11 11 10 9 8 10 8 9 30/30 Write block 64 len 20480 data: 5 7 9 9 8 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:13.126092596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 197 197 197 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 112 112 112 199 199 199 F F F 3 060-079 111 111 111 197 197 197 T T T 4 080-099 102 102 102 179 179 179 T T T 5 100-119 96 96 96 169 169 169 T T T 6 120-139 112 112 112 199 199 199 T T T 7 140-159 105 105 105 184 184 184 T T T 8 160-179 99 99 99 172 172 172 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 111 111 111 197 197 197 F F F 11 220-239 111 111 111 197 197 197 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 112 112 112 199 199 199 F F F 17 340-359 112 112 112 199 199 199 F F F 18 360-379 112 112 112 199 199 199 F F F 19 380-399 109 109 109 195 195 195 F F F 20 400-419 111 111 111 197 197 197 F F F 21 420-439 108 108 108 192 192 192 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 111 111 111 197 197 197 T T T 24 480-499 111 111 111 197 197 197 F F F 25 500-519 109 109 109 195 195 195 F F F 26 520-539 111 111 111 197 197 197 F F F 27 540-559 111 111 111 197 197 197 T T T 28 560-579 112 112 112 199 199 199 T T T 29 580-599 112 112 112 199 199 199 F F F Max gen: 112, Max flush: 199 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 111 111 111 197 197 197 F F F 1 020-039 104 104 104 183 183 183 F F F 2 040-059 112 112 112 199 199 199 F F F 3 060-079 113 113 113 200 200 200 F F F 4 080-099 113 113 113 200 200 200 F F F 5 100-119 113 113 113 200 200 200 F F F 6 120-139 113 113 113 200 200 200 F F F 7 140-159 113 113 113 200 200 200 F F F 8 160-179 99 99 99 172 172 172 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 111 111 111 197 197 197 F F F 11 220-239 111 111 111 197 197 197 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 112 112 112 199 199 199 F F F 17 340-359 112 112 112 199 199 199 F F F 18 360-379 112 112 112 199 199 199 F F F 19 380-399 109 109 109 195 195 195 F F F 20 400-419 111 111 111 197 197 197 F F F 21 420-439 108 108 108 192 192 192 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 113 113 113 200 200 200 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 109 109 109 195 195 195 F F F 26 520-539 111 111 111 197 197 197 F F F 27 540-559 113 113 113 200 200 200 F F F 28 560-579 113 113 113 200 200 200 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 113, Max flush: 200 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.407572155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.452070912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.452665797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.452711357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.452726927Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.452741637Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} {"msg":"Crucible b843f955-5f14-482f-ac91-c0c02d42a9cc has session id: b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.453224593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Upstairs opts: Upstairs UUID: b843f955-5f14-482f-ac91-c0c02d42a9cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.453275723Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Crucible stats registered with UUID: b843f955-5f14-482f-ac91-c0c02d42a9cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.453302333Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:17.453325272Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.453862238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.453892508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} {"msg":"b843f955-5f14-482f-ac91-c0c02d42a9cc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.453962177Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.453989147Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"0","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454042827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"0","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454079687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"1","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454111387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"1","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454140086Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"2","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454168076Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"2","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454193866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454219766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454246706Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51599)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454271735Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54748)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454299075Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38885)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.454325955Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.45492981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"1","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.45497301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"0","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.455507706Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"2","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.455625875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"1","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.455666325Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"0","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456129421Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"2","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.45632151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456412859Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[0]R flush_numbers[0..12]: [197, 183, 199, 200, 200, 200, 200, 200, 172, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456646728Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[0]R generation[0..12]: [111, 104, 112, 113, 113, 113, 113, 113, 99, 109, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456671857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456688567Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[1]R flush_numbers[0..12]: [197, 183, 199, 200, 200, 200, 200, 200, 172, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456703217Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[1]R generation[0..12]: [111, 104, 112, 113, 113, 113, 113, 113, 99, 109, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456718237Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456732927Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[2]R flush_numbers[0..12]: [197, 183, 199, 200, 200, 200, 200, 200, 172, 195, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456747807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[2]R generation[0..12]: [111, 104, 112, 113, 113, 113, 113, 113, 99, 109, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456761387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456778407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456791546Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456804696Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456817636Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456830206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"":"downstairs","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456843266Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456855046Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"0","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456868396Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"1","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456881446Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"client":"2","":"downstairs","session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456894536Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Wait for a query_work_queue command to finish before sending IO Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456913166Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"b843f955-5f14-482f-ac91-c0c02d42a9cc is now active with session: b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456926155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456938175Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648,"session_id":"b781bfdd-b0ab-46a9-bcbc-d2e7d446bd4a"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456950295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} {"msg":"Activated sub_volume b843f955-5f14-482f-ac91-c0c02d42a9cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:17.456961425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 353 len 36864 data: 14 16 12 12 12 10 12 9 7 02/30 Flush 03/30 Flush 04/30 Write block 208 len 16384 data: 12 11 12 13 05/30 Write block 425 len 36864 data: 9 8 8 7 10 10 14 11 12 06/30 Write block 393 len 20480 data: 4 3 3 3 3 07/30 Read block 85 len 20480 08/30 Write block 389 len 40960 data: 8 6 6 6 5 4 4 4 4 4 09/30 Read block 421 len 32768 10/30 Write block 16 len 36864 data: 14 15 13 14 12 14 13 16 14 11/30 Write block 173 len 16384 data: 8 7 8 7 12/30 Write block 87 len 12288 data: 8 8 9 13/30 Write block 218 len 16384 data: 8 8 8 9 14/30 Read block 506 len 4096 15/30 Read block 388 len 8192 16/30 Read block 378 len 40960 17/30 Write block 519 len 24576 data: 12 12 15 15 11 10 18/30 Flush 19/30 Write block 329 len 20480 data: 9 10 8 8 6 20/30 Read block 327 len 16384 21/30 Read block 164 len 40960 22/30 Write block 75 len 4096 data: 11 23/30 Write block 410 len 16384 data: 10 10 9 9 24/30 Write block 435 len 16384 data: 12 9 9 11 25/30 Write block 23 len 4096 data: 17 26/30 Read block 472 len 40960 27/30 Read block 289 len 20480 28/30 Write block 572 len 20480 data: 8 7 6 8 10 29/30 Read block 160 len 4096 30/30 Read block 86 len 36864 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 11 12 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:17.477141384Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 114 114 114 203 203 203 T T T 2 040-059 112 112 112 199 199 199 F F F 3 060-079 113 113 113 200 200 200 T T T 4 080-099 114 114 114 203 203 203 F F F 5 100-119 113 113 113 200 200 200 F F F 6 120-139 113 113 113 200 200 200 F F F 7 140-159 113 113 113 200 200 200 F F F 8 160-179 114 114 114 203 203 203 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 114 114 114 203 203 203 F F F 11 220-239 114 114 114 203 203 203 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 112 112 112 199 199 199 T T T 17 340-359 114 114 114 201 201 201 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 114 114 114 203 203 203 F F F 20 400-419 111 111 111 197 197 197 T T T 21 420-439 114 114 114 203 203 203 T T T 22 440-459 108 108 108 192 192 192 F F F 23 460-479 113 113 113 200 200 200 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 114 114 114 203 203 203 F F F 26 520-539 114 114 114 203 203 203 F F F 27 540-559 113 113 113 200 200 200 F F F 28 560-579 113 113 113 200 200 200 T T T 29 580-599 112 112 112 199 199 199 F F F Max gen: 114, Max flush: 203 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 112 112 112 199 199 199 F F F 3 060-079 115 115 115 204 204 204 F F F 4 080-099 114 114 114 203 203 203 F F F 5 100-119 113 113 113 200 200 200 F F F 6 120-139 113 113 113 200 200 200 F F F 7 140-159 113 113 113 200 200 200 F F F 8 160-179 114 114 114 203 203 203 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 114 114 114 203 203 203 F F F 11 220-239 114 114 114 203 203 203 F F F 12 240-259 106 106 106 187 187 187 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 115 115 115 204 204 204 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 114 114 114 203 203 203 F F F 20 400-419 115 115 115 204 204 204 F F F 21 420-439 115 115 115 204 204 204 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 113 113 113 200 200 200 F F F 24 480-499 111 111 111 197 197 197 F F F 25 500-519 114 114 114 203 203 203 F F F 26 520-539 114 114 114 203 203 203 F F F 27 540-559 113 113 113 200 200 200 F F F 28 560-579 115 115 115 204 204 204 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 115, Max flush: 204 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.717685809Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.761043174Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76160795Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76165422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.761670139Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.761685169Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} {"msg":"Crucible 8e843d95-5775-4fdc-8b77-e662ec14e3a6 has session id: 58f14fc1-db71-454c-8e44-cadf1b694b9e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762244415Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Upstairs opts: Upstairs UUID: 8e843d95-5775-4fdc-8b77-e662ec14e3a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762283825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Crucible stats registered with UUID: 8e843d95-5775-4fdc-8b77-e662ec14e3a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762307475Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:21.762327934Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762709892Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762740042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} {"msg":"8e843d95-5775-4fdc-8b77-e662ec14e3a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762814561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762832781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"0","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762862001Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"0","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.762877341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"1","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76289145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"1","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76290466Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"2","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76291852Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"2","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76293154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76294829Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.76296285Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48500)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.763093359Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42520)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.763135499Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38338)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.763163008Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.763916343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"0","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.763956732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"1","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.764587758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"2","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.764752586Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"1","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.764782556Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"0","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765168553Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"2","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765415822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765533661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[0]R flush_numbers[0..12]: [203, 204, 199, 204, 203, 200, 200, 200, 203, 195, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765812859Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[0]R generation[0..12]: [114, 115, 112, 115, 114, 113, 113, 113, 114, 109, 114, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765832868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765852708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[1]R flush_numbers[0..12]: [203, 204, 199, 204, 203, 200, 200, 200, 203, 195, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765867118Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[1]R generation[0..12]: [114, 115, 112, 115, 114, 113, 113, 113, 114, 109, 114, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765880638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765894958Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[2]R flush_numbers[0..12]: [203, 204, 199, 204, 203, 200, 200, 200, 203, 195, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765908738Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[2]R generation[0..12]: [114, 115, 112, 115, 114, 113, 113, 113, 114, 109, 114, 114]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765922398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765936447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765949538Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765970367Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.765994347Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"All extents match","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-03-18T05:05:21.766024717Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"":"downstairs","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766050787Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766072757Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"0","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766087066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"1","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766106596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"client":"2","":"downstairs","session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766130726Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766154366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"8e843d95-5775-4fdc-8b77-e662ec14e3a6 is now active with session: 58f14fc1-db71-454c-8e44-cadf1b694b9e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766178236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766191876Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656,"session_id":"58f14fc1-db71-454c-8e44-cadf1b694b9e"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766206296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} {"msg":"Activated sub_volume 8e843d95-5775-4fdc-8b77-e662ec14e3a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:21.766225865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 465 len 12288 data: 9 7 8 02/30 Read block 422 len 12288 03/30 Flush 04/30 Write block 143 len 40960 data: 10 8 7 6 6 8 9 10 11 12 05/30 Write block 385 len 24576 data: 9 11 8 7 9 7 06/30 Write block 224 len 4096 data: 7 07/30 Read block 553 len 12288 08/30 Write block 525 len 12288 data: 9 11 9 09/30 Read block 314 len 16384 10/30 Write block 413 len 28672 data: 10 6 6 6 4 4 6 11/30 Read block 227 len 20480 12/30 Read block 251 len 36864 13/30 Read block 567 len 20480 14/30 Write block 58 len 32768 data: 9 9 8 8 5 5 6 8 15/30 Write block 250 len 20480 data: 10 8 7 8 12 16/30 Read block 323 len 4096 17/30 Read block 309 len 16384 18/30 Read block 586 len 12288 19/30 Write block 174 len 20480 data: 8 9 8 7 7 20/30 Write block 482 len 16384 data: 11 13 13 13 21/30 Read block 545 len 8192 22/30 Flush 23/30 Write block 210 len 36864 data: 13 14 12 10 12 11 10 10 9 24/30 Read block 167 len 20480 25/30 Read block 57 len 4096 26/30 Flush 27/30 Write block 529 len 24576 data: 7 6 7 6 7 7 28/30 Read block 534 len 28672 29/30 Read block 311 len 8192 30/30 Write block 95 len 8192 data: 12 11 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:21.786580793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 116 116 116 206 206 206 F F F 4 080-099 114 114 114 203 203 203 T T T 5 100-119 113 113 113 200 200 200 F F F 6 120-139 113 113 113 200 200 200 F F F 7 140-159 116 116 116 206 206 206 F F F 8 160-179 116 116 116 206 206 206 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 116 116 116 207 207 207 F F F 11 220-239 116 116 116 206 206 206 F F F 12 240-259 116 116 116 206 206 206 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 115 115 115 204 204 204 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 116 116 116 206 206 206 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 115 115 115 204 204 204 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 116 116 116 205 205 205 F F F 24 480-499 116 116 116 206 206 206 F F F 25 500-519 114 114 114 203 203 203 F F F 26 520-539 116 116 116 206 206 206 T T T 27 540-559 113 113 113 200 200 200 F F F 28 560-579 115 115 115 204 204 204 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 116, Max flush: 207 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 116 116 116 206 206 206 F F F 4 080-099 117 117 117 208 208 208 F F F 5 100-119 113 113 113 200 200 200 F F F 6 120-139 113 113 113 200 200 200 F F F 7 140-159 116 116 116 206 206 206 F F F 8 160-179 116 116 116 206 206 206 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 116 116 116 207 207 207 F F F 11 220-239 116 116 116 206 206 206 F F F 12 240-259 116 116 116 206 206 206 F F F 13 260-279 106 106 106 185 185 185 F F F 14 280-299 108 108 108 194 194 194 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 115 115 115 204 204 204 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 116 116 116 206 206 206 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 115 115 115 204 204 204 F F F 22 440-459 108 108 108 192 192 192 F F F 23 460-479 116 116 116 205 205 205 F F F 24 480-499 116 116 116 206 206 206 F F F 25 500-519 114 114 114 203 203 203 F F F 26 520-539 117 117 117 208 208 208 F F F 27 540-559 113 113 113 200 200 200 F F F 28 560-579 115 115 115 204 204 204 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 117, Max flush: 208 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.008788897Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:26.051287739Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.051644406Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.082321936Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.082837002Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.082874752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.082890742Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.082908362Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} {"msg":"Crucible b9eadb42-a5fe-4049-929f-97d7f0fd5bfa has session id: 980d502d-d7fa-47a6-bbb8-85d4f3aedf92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.083448898Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Upstairs opts: Upstairs UUID: b9eadb42-a5fe-4049-929f-97d7f0fd5bfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.083488208Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Crucible stats registered with UUID: b9eadb42-a5fe-4049-929f-97d7f0fd5bfa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.083513157Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:26.083529757Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.083927084Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.083951194Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} {"msg":"b9eadb42-a5fe-4049-929f-97d7f0fd5bfa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084034324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084052384Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"0","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084083813Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"0","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084099283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"1","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084113693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"1","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084126593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"2","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084202042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"2","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084233322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084261682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084289832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"ds_connection connected from Ok(127.0.0.1:57734)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084316402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"ds_connection connected from Ok(127.0.0.1:33290)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.084343481Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"ds_connection connected from Ok(127.0.0.1:58271)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.08444249Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.085072646Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"2","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.085101565Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"0","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.085117516Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"1","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.085676801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"2","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.085758601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"1","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.085788631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"0","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086275837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086394266Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[0]R flush_numbers[0..12]: [203, 204, 206, 206, 208, 200, 200, 206, 206, 195, 207, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086431226Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[0]R generation[0..12]: [114, 115, 116, 116, 117, 113, 113, 116, 116, 109, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086447555Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086462015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[1]R flush_numbers[0..12]: [203, 204, 206, 206, 208, 200, 200, 206, 206, 195, 207, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086475825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[1]R generation[0..12]: [114, 115, 116, 116, 117, 113, 113, 116, 116, 109, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086489275Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086503095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[2]R flush_numbers[0..12]: [203, 204, 206, 206, 208, 200, 200, 206, 206, 195, 207, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086516945Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[2]R generation[0..12]: [114, 115, 116, 116, 117, 113, 113, 116, 116, 109, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086530625Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086544485Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086561675Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086575185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086588025Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086601204Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086621734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086634054Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"0","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086647274Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"1","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086660134Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"client":"2","":"downstairs","session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086672954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086685154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"b9eadb42-a5fe-4049-929f-97d7f0fd5bfa is now active with session: 980d502d-d7fa-47a6-bbb8-85d4f3aedf92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086697894Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086709844Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664,"session_id":"980d502d-d7fa-47a6-bbb8-85d4f3aedf92"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086721933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} {"msg":"Activated sub_volume b9eadb42-a5fe-4049-929f-97d7f0fd5bfa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:26.086732953Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 527 len 12288 data: 10 7 8 02/30 Read block 572 len 40960 03/30 Write block 121 len 20480 data: 9 10 11 13 12 04/30 Read block 45 len 28672 05/30 Read block 359 len 36864 06/30 Read block 503 len 36864 07/30 Read block 372 len 28672 08/30 Write block 457 len 12288 data: 9 5 6 09/30 Read block 73 len 16384 10/30 Write block 387 len 32768 data: 9 8 10 8 7 7 6 5 11/30 Read block 459 len 32768 12/30 Read block 47 len 20480 13/30 Read block 166 len 32768 14/30 Write block 275 len 36864 data: 9 5 3 5 6 7 8 7 6 15/30 Read block 208 len 12288 16/30 Read block 348 len 20480 17/30 Write block 492 len 40960 data: 17 15 14 12 8 10 11 10 9 9 18/30 Write block 544 len 8192 data: 11 13 19/30 Read block 353 len 4096 20/30 Read block 113 len 32768 21/30 Read block 446 len 16384 22/30 Read block 131 len 40960 23/30 Write block 119 len 12288 data: 11 11 10 24/30 Write block 69 len 4096 data: 9 25/30 Write block 136 len 28672 data: 12 11 10 9 11 9 10 26/30 Read block 359 len 40960 27/30 Read block 136 len 40960 28/30 Read block 226 len 8192 29/30 Write block 209 len 32768 data: 12 14 15 13 11 13 12 11 30/30 Write block 210 len 40960 data: 15 16 14 12 14 13 12 11 10 9 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Sent Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 28 29 28 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:26.107981164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 116 116 116 206 206 206 T T T 4 080-099 117 117 117 208 208 208 F F F 5 100-119 113 113 113 200 200 200 T T T 6 120-139 113 113 113 200 200 200 T T T 7 140-159 116 116 116 206 206 206 T T T 8 160-179 116 116 116 206 206 206 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 116 116 116 207 207 207 T T T 11 220-239 116 116 116 206 206 206 F F F 12 240-259 116 116 116 206 206 206 F F F 13 260-279 106 106 106 185 185 185 T T T 14 280-299 108 108 108 194 194 194 T T T 15 300-319 107 107 107 188 188 188 F F F 16 320-339 115 115 115 204 204 204 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 116 116 116 206 206 206 T T T 20 400-419 116 116 116 206 206 206 F F F 21 420-439 115 115 115 204 204 204 F F F 22 440-459 108 108 108 192 192 192 T T T 23 460-479 116 116 116 205 205 205 F F F 24 480-499 116 116 116 206 206 206 T T T 25 500-519 114 114 114 203 203 203 T T T 26 520-539 117 117 117 208 208 208 T T T 27 540-559 113 113 113 200 200 200 T T T 28 560-579 115 115 115 204 204 204 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 117, Max flush: 208 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 115 115 115 204 204 204 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 118 118 118 209 209 209 F F F 4 080-099 117 117 117 208 208 208 F F F 5 100-119 118 118 118 209 209 209 F F F 6 120-139 118 118 118 209 209 209 F F F 7 140-159 118 118 118 209 209 209 F F F 8 160-179 116 116 116 206 206 206 F F F 9 180-199 109 109 109 195 195 195 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 116 116 116 206 206 206 F F F 12 240-259 116 116 116 206 206 206 F F F 13 260-279 118 118 118 209 209 209 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 115 115 115 204 204 204 F F F 17 340-359 114 114 114 201 201 201 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 118 118 118 209 209 209 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 115 115 115 204 204 204 F F F 22 440-459 118 118 118 209 209 209 F F F 23 460-479 116 116 116 205 205 205 F F F 24 480-499 118 118 118 209 209 209 F F F 25 500-519 118 118 118 209 209 209 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 115 115 115 204 204 204 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 118, Max flush: 209 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.416707Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.460103944Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46068151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46072543Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46074397Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46075942Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} {"msg":"Crucible c2b81460-244c-4fdc-9371-e7d3159ae06e has session id: 8fa547d5-bf16-42f8-8445-e873c78d4a5c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.461275836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Upstairs opts: Upstairs UUID: c2b81460-244c-4fdc-9371-e7d3159ae06e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.461316075Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Crucible stats registered with UUID: c2b81460-244c-4fdc-9371-e7d3159ae06e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.461339875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:30.461360545Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.461882021Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.461905331Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} {"msg":"c2b81460-244c-4fdc-9371-e7d3159ae06e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46198705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46201129Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"0","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46204969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"0","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46207682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"1","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.46210508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"1","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462129769Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"2","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462156169Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"2","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462179369Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462203579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462230959Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63464)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462255248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39811)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462372098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33085)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.462425257Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.463001873Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"2","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.463047473Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"0","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.463641328Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"0","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.463772287Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"2","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.463869496Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"1","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.464160244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.464312233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.464459152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"1","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[0]R flush_numbers[0..12]: [203, 204, 206, 209, 208, 209, 209, 209, 206, 195, 209, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.464960708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[0]R generation[0..12]: [114, 115, 116, 118, 117, 118, 118, 118, 116, 109, 118, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.464982678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465011308Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[1]R flush_numbers[0..12]: [203, 204, 206, 209, 208, 209, 209, 209, 206, 195, 209, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465027708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[1]R generation[0..12]: [114, 115, 116, 118, 117, 118, 118, 118, 116, 109, 118, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465041738Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465055638Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[2]R flush_numbers[0..12]: [203, 204, 206, 209, 208, 209, 209, 209, 206, 195, 209, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465070057Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[2]R generation[0..12]: [114, 115, 116, 118, 117, 118, 118, 118, 116, 109, 118, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465083767Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465097417Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Max found gen is 119","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465110007Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Generation requested: 120 >= found:119","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465123577Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465143017Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465155947Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"":"downstairs","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465172357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465184846Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"0","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465198037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"1","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465210956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"client":"2","":"downstairs","session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465223936Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465236126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"c2b81460-244c-4fdc-9371-e7d3159ae06e is now active with session: 8fa547d5-bf16-42f8-8445-e873c78d4a5c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465248916Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465260676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672,"session_id":"8fa547d5-bf16-42f8-8445-e873c78d4a5c"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465272876Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} {"msg":"Activated sub_volume c2b81460-244c-4fdc-9371-e7d3159ae06e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:30.465284196Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 20480 data: 7 8 10 11 12 02/30 Read block 71 len 24576 03/30 Read block 445 len 24576 04/30 Write block 349 len 12288 data: 11 11 16 05/30 Read block 271 len 24576 06/30 Write block 100 len 8192 data: 9 6 07/30 Write block 242 len 40960 data: 6 7 7 6 6 7 11 11 11 9 08/30 Read block 586 len 40960 09/30 Write block 123 len 12288 data: 12 14 13 10/30 Flush 11/30 Write block 184 len 28672 data: 9 9 8 7 5 6 7 12/30 Write block 230 len 40960 data: 10 7 6 5 4 7 6 5 6 7 13/30 Read block 457 len 12288 14/30 Write block 425 len 32768 data: 10 9 9 8 11 11 15 12 15/30 Read block 58 len 4096 16/30 Read block 56 len 40960 17/30 Read block 498 len 24576 18/30 Read block 584 len 36864 19/30 Read block 521 len 4096 20/30 Write block 31 len 28672 data: 8 9 11 12 13 14 13 21/30 Read block 75 len 24576 22/30 Read block 502 len 4096 23/30 Flush 24/30 Write block 560 len 36864 data: 13 13 9 8 8 9 7 6 8 25/30 Read block 22 len 24576 26/30 Read block 40 len 24576 27/30 Write block 90 len 20480 data: 12 11 13 14 15 28/30 Read block 411 len 12288 29/30 Read block 503 len 28672 30/30 Write block 231 len 32768 data: 8 7 6 5 8 7 6 7 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:30.486682205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 118 118 118 209 209 209 F F F 4 080-099 117 117 117 208 208 208 T T T 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 118 118 118 209 209 209 F F F 8 160-179 116 116 116 206 206 206 F F F 9 180-199 120 120 120 211 211 211 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 120 120 120 211 211 211 T T T 12 240-259 120 120 120 210 210 210 F F F 13 260-279 118 118 118 209 209 209 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 115 115 115 204 204 204 F F F 17 340-359 120 120 120 210 210 210 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 118 118 118 209 209 209 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 120 120 120 211 211 211 F F F 22 440-459 118 118 118 209 209 209 F F F 23 460-479 116 116 116 205 205 205 F F F 24 480-499 118 118 118 209 209 209 F F F 25 500-519 118 118 118 209 209 209 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 115 115 115 204 204 204 T T T 29 580-599 112 112 112 199 199 199 F F F Max gen: 120, Max flush: 211 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 118 118 118 209 209 209 F F F 4 080-099 121 121 121 212 212 212 F F F 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 118 118 118 209 209 209 F F F 8 160-179 116 116 116 206 206 206 F F F 9 180-199 120 120 120 211 211 211 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 121 121 121 212 212 212 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 118 118 118 209 209 209 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 107 107 107 188 188 188 F F F 16 320-339 115 115 115 204 204 204 F F F 17 340-359 120 120 120 210 210 210 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 118 118 118 209 209 209 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 120 120 120 211 211 211 F F F 22 440-459 118 118 118 209 209 209 F F F 23 460-479 116 116 116 205 205 205 F F F 24 480-499 118 118 118 209 209 209 F F F 25 500-519 118 118 118 209 209 209 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 121 121 121 212 212 212 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 121, Max flush: 212 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.71140571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.754859904Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.755718588Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.755770938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.755788888Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.755806897Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} {"msg":"Crucible c095c29d-fb2b-4566-a909-3ac6bd481c17 has session id: 9074907d-0560-408f-8c91-7b454f27f5e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.756435163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Upstairs opts: Upstairs UUID: c095c29d-fb2b-4566-a909-3ac6bd481c17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.756476572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Crucible stats registered with UUID: c095c29d-fb2b-4566-a909-3ac6bd481c17","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.756497792Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:34.756511242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.756992979Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757026688Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} {"msg":"c095c29d-fb2b-4566-a909-3ac6bd481c17 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757113438Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757132078Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"0","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757158807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"0","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757172657Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"1","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757186917Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"1","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757206137Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757232427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757255777Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757270457Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757344366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:34.757382305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:34.757411705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35827)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757439245Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:34.757469035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48271)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757502875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.757607954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.758013451Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"1","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.758195259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"0","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.758541017Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"1","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.758652806Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"0","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.759156432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.759246162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.758390757Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53081)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.758523426Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.759876746Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.760486911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[0]R flush_numbers[0..12]: [203, 211, 206, 209, 212, 210, 210, 209, 206, 211, 209, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761081107Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[0]R generation[0..12]: [114, 120, 116, 118, 121, 120, 120, 118, 116, 120, 118, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761104887Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761124836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[1]R flush_numbers[0..12]: [203, 211, 206, 209, 212, 210, 210, 209, 206, 211, 209, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761139276Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[1]R generation[0..12]: [114, 120, 116, 118, 121, 120, 120, 118, 116, 120, 118, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761153026Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761166976Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[2]R flush_numbers[0..12]: [203, 211, 206, 209, 212, 210, 210, 209, 206, 211, 209, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761181926Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[2]R generation[0..12]: [114, 120, 116, 118, 121, 120, 120, 118, 116, 120, 118, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761195486Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761209306Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761234575Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761265655Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761289335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761305815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"":"downstairs","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761327035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761353955Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"0","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761388734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"1","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761418944Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"client":"2","":"downstairs","session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761454914Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761485424Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"c095c29d-fb2b-4566-a909-3ac6bd481c17 is now active with session: 9074907d-0560-408f-8c91-7b454f27f5e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761530853Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761562703Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680,"session_id":"9074907d-0560-408f-8c91-7b454f27f5e5"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761592483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} {"msg":"Activated sub_volume c095c29d-fb2b-4566-a909-3ac6bd481c17","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:44.761620903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 381 len 24576 data: 7 6 6 8 10 12 02/30 Read block 182 len 8192 03/30 Write block 462 len 40960 data: 9 10 8 10 8 9 8 8 10 8 04/30 Read block 83 len 16384 05/30 Write block 81 len 16384 data: 7 6 5 7 06/30 Read block 176 len 40960 07/30 Read block 63 len 36864 08/30 Write block 383 len 16384 data: 7 9 11 13 09/30 Write block 356 len 4096 data: 13 10/30 Read block 434 len 8192 11/30 Flush 12/30 Write block 176 len 36864 data: 9 8 8 9 9 8 9 9 10 13/30 Read block 156 len 36864 14/30 Write block 504 len 8192 data: 9 9 15/30 Read block 221 len 28672 16/30 Read block 256 len 4096 17/30 Read block 423 len 24576 18/30 Write block 184 len 32768 data: 11 10 9 8 6 7 8 7 19/30 Write block 430 len 36864 data: 12 16 13 13 13 13 10 10 12 20/30 Read block 235 len 16384 21/30 Write block 318 len 36864 data: 8 9 12 14 10 11 13 13 15 22/30 Write block 384 len 28672 data: 10 12 14 10 9 11 9 23/30 Read block 566 len 40960 24/30 Write block 234 len 16384 data: 6 9 8 7 25/30 Read block 508 len 36864 26/30 Write block 182 len 40960 data: 10 10 12 11 10 9 7 8 9 8 27/30 Read block 28 len 8192 28/30 Write block 145 len 16384 data: 8 7 7 9 29/30 Read block 571 len 4096 30/30 Write block 335 len 12288 data: 9 10 12 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:44.783457249Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 118 118 118 209 209 209 F F F 4 080-099 122 122 122 213 213 213 F F F 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 118 118 118 209 209 209 T T T 8 160-179 116 116 116 206 206 206 T T T 9 180-199 120 120 120 211 211 211 T T T 10 200-219 118 118 118 209 209 209 F F F 11 220-239 121 121 121 212 212 212 T T T 12 240-259 120 120 120 210 210 210 F F F 13 260-279 118 118 118 209 209 209 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 107 107 107 188 188 188 T T T 16 320-339 115 115 115 204 204 204 T T T 17 340-359 122 122 122 213 213 213 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 122 122 122 213 213 213 T T T 20 400-419 116 116 116 206 206 206 F F F 21 420-439 120 120 120 211 211 211 T T T 22 440-459 118 118 118 209 209 209 F F F 23 460-479 122 122 122 213 213 213 F F F 24 480-499 118 118 118 209 209 209 F F F 25 500-519 118 118 118 209 209 209 T T T 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 121 121 121 212 212 212 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 122, Max flush: 213 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 F F F 1 020-039 120 120 120 211 211 211 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 118 118 118 209 209 209 F F F 4 080-099 122 122 122 213 213 213 F F F 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 F F F 8 160-179 123 123 123 214 214 214 F F F 9 180-199 123 123 123 214 214 214 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 123 123 123 214 214 214 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 118 118 118 209 209 209 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 123 123 123 214 214 214 F F F 16 320-339 123 123 123 214 214 214 F F F 17 340-359 122 122 122 213 213 213 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 123 123 123 214 214 214 F F F 20 400-419 116 116 116 206 206 206 F F F 21 420-439 123 123 123 214 214 214 F F F 22 440-459 118 118 118 209 209 209 F F F 23 460-479 122 122 122 213 213 213 F F F 24 480-499 118 118 118 209 209 209 F F F 25 500-519 123 123 123 214 214 214 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 121 121 121 212 212 212 F F F 29 580-599 112 112 112 199 199 199 F F F Max gen: 123, Max flush: 214 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.045620232Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.089405704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.08996066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09003721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09005991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.090082479Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} {"msg":"Crucible 1a6d60cf-f2af-4ce0-b28e-15e6869cfc4f has session id: 90d81a92-af21-4a5a-94b2-24136e0499dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.090539326Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Upstairs opts: Upstairs UUID: 1a6d60cf-f2af-4ce0-b28e-15e6869cfc4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.090582535Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Crucible stats registered with UUID: 1a6d60cf-f2af-4ce0-b28e-15e6869cfc4f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.090598095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:49.090611025Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091096562Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091121802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} {"msg":"1a6d60cf-f2af-4ce0-b28e-15e6869cfc4f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091203341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091224591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"0","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09125283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"0","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09126695Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"1","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09128107Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"1","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09133524Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"2","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09136824Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"2","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091395489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091420239Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091445909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:46981)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091472259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:57664)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091496599Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:64029)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.091626588Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.092086294Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"1","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.092125244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"0","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09270879Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"1","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.092748039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"0","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.093347455Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.093382294Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.09393129Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"2","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.094443057Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"2","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[0]R flush_numbers[0..12]: [203, 211, 206, 209, 213, 210, 210, 214, 214, 214, 209, 214]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.094932283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[0]R generation[0..12]: [114, 120, 116, 118, 122, 120, 120, 123, 123, 123, 118, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.094955003Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.094970293Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[1]R flush_numbers[0..12]: [203, 211, 206, 209, 213, 210, 210, 214, 214, 214, 209, 214]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.094984402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[1]R generation[0..12]: [114, 120, 116, 118, 122, 120, 120, 123, 123, 123, 118, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.094998162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095012312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[2]R flush_numbers[0..12]: [203, 211, 206, 209, 213, 210, 210, 214, 214, 214, 209, 214]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095026202Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[2]R generation[0..12]: [114, 120, 116, 118, 122, 120, 120, 123, 123, 123, 118, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095039512Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095056542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095069862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095082892Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095095692Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095114172Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"":"downstairs","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095127432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095139411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"0","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095152531Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"1","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095166561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"client":"2","":"downstairs","session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095179751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095192631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"1a6d60cf-f2af-4ce0-b28e-15e6869cfc4f is now active with session: 90d81a92-af21-4a5a-94b2-24136e0499dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095205311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095217341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688,"session_id":"90d81a92-af21-4a5a-94b2-24136e0499dd"} {"msg":"The guest has finished waiting for activation with:124","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095232851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} {"msg":"Activated sub_volume 1a6d60cf-f2af-4ce0-b28e-15e6869cfc4f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:49.095244191Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 221 len 24576 data: 10 8 8 8 9 8 02/30 Flush 03/30 Write block 417 len 28672 data: 5 5 7 6 9 9 10 04/30 Write block 503 len 4096 data: 9 05/30 Write block 382 len 40960 data: 7 8 11 13 15 11 10 12 10 8 06/30 Read block 154 len 8192 07/30 Read block 500 len 40960 08/30 Flush 09/30 Write block 578 len 16384 data: 11 11 13 10 10/30 Flush 11/30 Write block 26 len 24576 data: 14 11 9 5 5 9 12/30 Write block 81 len 40960 data: 8 7 6 8 7 7 9 9 10 13 13/30 Read block 236 len 28672 14/30 Write block 270 len 36864 data: 11 11 11 10 8 10 6 4 6 15/30 Read block 519 len 12288 16/30 Read block 87 len 28672 17/30 Read block 561 len 12288 18/30 Read block 325 len 4096 19/30 Write block 12 len 8192 data: 13 12 20/30 Read block 186 len 28672 21/30 Read block 398 len 8192 22/30 Read block 545 len 32768 23/30 Read block 84 len 20480 24/30 Read block 297 len 4096 25/30 Write block 348 len 36864 data: 9 12 12 17 16 15 17 13 14 26/30 Write block 318 len 28672 data: 9 10 13 15 11 12 14 27/30 Write block 385 len 28672 data: 14 16 12 11 13 11 9 28/30 Read block 427 len 16384 29/30 Read block 515 len 24576 30/30 Read block 363 len 36864 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 6 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:49.115865226Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 203 203 203 T T T 1 020-039 120 120 120 211 211 211 T T T 2 040-059 116 116 116 206 206 206 F F F 3 060-079 118 118 118 209 209 209 F F F 4 080-099 122 122 122 213 213 213 T T T 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 F F F 8 160-179 123 123 123 214 214 214 F F F 9 180-199 123 123 123 214 214 214 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 118 118 118 209 209 209 T T T 14 280-299 118 118 118 209 209 209 F F F 15 300-319 123 123 123 214 214 214 T T T 16 320-339 123 123 123 214 214 214 T T T 17 340-359 122 122 122 213 213 213 T T T 18 360-379 114 114 114 201 201 201 F F F 19 380-399 124 124 124 216 216 216 T T T 20 400-419 124 124 124 216 216 216 F F F 21 420-439 124 124 124 216 216 216 F F F 22 440-459 118 118 118 209 209 209 F F F 23 460-479 122 122 122 213 213 213 F F F 24 480-499 118 118 118 209 209 209 F F F 25 500-519 124 124 124 216 216 216 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 124 124 124 217 217 217 F F F 29 580-599 124 124 124 217 217 217 F F F Max gen: 124, Max flush: 217 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 125 125 125 218 218 218 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 118 118 118 209 209 209 F F F 4 080-099 125 125 125 218 218 218 F F F 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 F F F 8 160-179 123 123 123 214 214 214 F F F 9 180-199 123 123 123 214 214 214 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 125 125 125 218 218 218 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 125 125 125 218 218 218 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 125 125 125 218 218 218 F F F 20 400-419 124 124 124 216 216 216 F F F 21 420-439 124 124 124 216 216 216 F F F 22 440-459 118 118 118 209 209 209 F F F 23 460-479 122 122 122 213 213 213 F F F 24 480-499 118 118 118 209 209 209 F F F 25 500-519 124 124 124 216 216 216 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 124 124 124 217 217 217 F F F 29 580-599 124 124 124 217 217 217 F F F Max gen: 125, Max flush: 218 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.369927581Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.413316646Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.413935631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.413978691Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.414019801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.41403734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} {"msg":"Crucible c7d452e8-2bd3-44e0-a7ea-52c73491c468 has session id: 383edd70-4e8e-4379-b224-8ca8a4cef48e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.414539407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Upstairs opts: Upstairs UUID: c7d452e8-2bd3-44e0-a7ea-52c73491c468, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.414579736Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Crucible stats registered with UUID: c7d452e8-2bd3-44e0-a7ea-52c73491c468","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.414603236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:53.414624416Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415140802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415178792Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} {"msg":"c7d452e8-2bd3-44e0-a7ea-52c73491c468 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415225642Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415253982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"0","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415287521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"0","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415303891Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"1","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415318901Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"1","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415332571Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"2","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415346791Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"2","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.41536815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.41539069Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.41541528Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47734)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.41548116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61990)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415521509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48786)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.415549919Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.416220134Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"1","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.416249744Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"0","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.416370973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"2","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.416897089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"1","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.416937109Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"2","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.416967278Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"0","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417484085Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417586144Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[0]R flush_numbers[0..12]: [218, 218, 206, 209, 218, 210, 210, 214, 214, 214, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417621054Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[0]R generation[0..12]: [125, 125, 116, 118, 125, 120, 120, 123, 123, 123, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417637173Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417651564Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[1]R flush_numbers[0..12]: [218, 218, 206, 209, 218, 210, 210, 214, 214, 214, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417665623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[1]R generation[0..12]: [125, 125, 116, 118, 125, 120, 120, 123, 123, 123, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417679343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417693163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[2]R flush_numbers[0..12]: [218, 218, 206, 209, 218, 210, 210, 214, 214, 214, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417706693Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[2]R generation[0..12]: [125, 125, 116, 118, 125, 120, 120, 123, 123, 123, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417720253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417733893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417746803Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417759903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417772552Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417791183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"":"downstairs","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417806762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417821142Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"0","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417834422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"1","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417847522Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"client":"2","":"downstairs","session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417860462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417872652Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"c7d452e8-2bd3-44e0-a7ea-52c73491c468 is now active with session: 383edd70-4e8e-4379-b224-8ca8a4cef48e","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:53.417885292Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417902502Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696,"session_id":"383edd70-4e8e-4379-b224-8ca8a4cef48e"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417914622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} {"msg":"Activated sub_volume c7d452e8-2bd3-44e0-a7ea-52c73491c468","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:53.417925751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 66 len 32768 data: 10 10 9 10 9 9 11 11 02/30 Read block 207 len 36864 03/30 Write block 68 len 32768 data: 10 11 10 10 12 12 10 12 04/30 Read block 457 len 8192 05/30 Read block 521 len 28672 06/30 Write block 169 len 24576 data: 10 10 7 7 9 9 07/30 Write block 455 len 16384 data: 9 9 10 6 08/30 Read block 406 len 32768 09/30 Read block 468 len 28672 10/30 Write block 401 len 36864 data: 9 10 13 13 12 11 11 10 11 11/30 Write block 489 len 28672 data: 13 13 14 18 16 15 13 12/30 Flush 13/30 Write block 20 len 28672 data: 13 15 14 18 15 12 15 14/30 Read block 476 len 32768 15/30 Write block 323 len 24576 data: 13 15 14 16 16 11 16/30 Flush 17/30 Write block 179 len 36864 data: 10 10 9 11 11 13 12 11 10 18/30 Read block 463 len 8192 19/30 Read block 349 len 16384 20/30 Write block 582 len 32768 data: 11 10 10 10 10 10 10 10 21/30 Read block 454 len 4096 22/30 Read block 284 len 4096 23/30 Read block 480 len 16384 24/30 Read block 409 len 28672 25/30 Read block 146 len 20480 26/30 Read block 422 len 32768 27/30 Write block 493 len 4096 data: 17 28/30 Read block 413 len 12288 29/30 Write block 462 len 12288 data: 10 11 9 30/30 Write block 75 len 32768 data: 13 11 10 11 10 7 9 8 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 9 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:53.439958677Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 126 126 126 220 220 220 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 126 126 126 219 219 219 T T T 4 080-099 125 125 125 218 218 218 T T T 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 F F F 8 160-179 126 126 126 219 219 219 T T T 9 180-199 123 123 123 214 214 214 T T T 10 200-219 118 118 118 209 209 209 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 125 125 125 218 218 218 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 125 125 125 218 218 218 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 124 124 124 216 216 216 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 122 122 122 213 213 213 T T T 24 480-499 126 126 126 219 219 219 T T T 25 500-519 124 124 124 216 216 216 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 124 124 124 217 217 217 F F F 29 580-599 124 124 124 217 217 217 T T T Max gen: 126, Max flush: 220 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 126 126 126 220 220 220 F F F 2 040-059 116 116 116 206 206 206 F F F 3 060-079 127 127 127 221 221 221 F F F 4 080-099 127 127 127 221 221 221 F F F 5 100-119 120 120 120 210 210 210 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 F F F 8 160-179 127 127 127 221 221 221 F F F 9 180-199 127 127 127 221 221 221 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 125 125 125 218 218 218 F F F 14 280-299 118 118 118 209 209 209 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 125 125 125 218 218 218 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 124 124 124 216 216 216 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 127 127 127 221 221 221 F F F 24 480-499 127 127 127 221 221 221 F F F 25 500-519 124 124 124 216 216 216 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 118 118 118 209 209 209 F F F 28 560-579 124 124 124 217 217 217 F F F 29 580-599 127 127 127 221 221 221 F F F Max gen: 127, Max flush: 221 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.693548405Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.737065479Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.737618095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.737659924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.737675124Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.737689634Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} {"msg":"Crucible 3a390b51-fc51-47bc-b6da-8a0d1f8d41a3 has session id: d7bf33c8-1a92-4765-a8c1-2a61b04d04c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.73818823Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Upstairs opts: Upstairs UUID: 3a390b51-fc51-47bc-b6da-8a0d1f8d41a3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738296939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Crucible stats registered with UUID: 3a390b51-fc51-47bc-b6da-8a0d1f8d41a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738329099Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:57.738350749Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738781296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738807566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} {"msg":"3a390b51-fc51-47bc-b6da-8a0d1f8d41a3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738861125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738886255Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"0","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738915225Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"0","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738930205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"1","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738944605Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"1","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738957665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"2","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738971585Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"2","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.738984524Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.739025284Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.739054874Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40942)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.739138623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:59772)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.739175103Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44549)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.739200053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.739865688Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"0","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.739910417Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"2","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.740564813Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"1","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.740614292Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"2","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.740682362Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"0","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741157358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"1","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741344127Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741364736Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[0]R flush_numbers[0..12]: [218, 220, 206, 221, 221, 210, 210, 214, 221, 221, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741696414Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[0]R generation[0..12]: [125, 126, 116, 127, 127, 120, 120, 123, 127, 127, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741732374Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741759164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[1]R flush_numbers[0..12]: [218, 220, 206, 221, 221, 210, 210, 214, 221, 221, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741785893Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[1]R generation[0..12]: [125, 126, 116, 127, 127, 120, 120, 123, 127, 127, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741810973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741837113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[2]R flush_numbers[0..12]: [218, 220, 206, 221, 221, 210, 210, 214, 221, 221, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741865783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[2]R generation[0..12]: [125, 126, 116, 127, 127, 120, 120, 123, 127, 127, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741898563Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:57.741927872Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741959192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.741987172Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.742014022Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.742040642Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"":"downstairs","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.742066661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.742090941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"0","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:05:57.742117131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"1","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.742143401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"client":"2","":"downstairs","session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.742168081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.7421821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"3a390b51-fc51-47bc-b6da-8a0d1f8d41a3 is now active with session: d7bf33c8-1a92-4765-a8c1-2a61b04d04c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.74219526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.74220727Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704,"session_id":"d7bf33c8-1a92-4765-a8c1-2a61b04d04c5"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.74221952Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} {"msg":"Activated sub_volume 3a390b51-fc51-47bc-b6da-8a0d1f8d41a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.74223517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 191 len 4096 data: 9 02/30 Write block 63 len 28672 data: 6 7 9 11 11 11 12 03/30 Read block 163 len 32768 04/30 Read block 545 len 12288 05/30 Write block 544 len 24576 data: 12 14 14 18 15 16 06/30 Read block 295 len 36864 07/30 Write block 278 len 40960 data: 7 7 8 9 8 7 9 12 11 13 08/30 Read block 354 len 32768 09/30 Flush 10/30 Write block 30 len 4096 data: 6 11/30 Read block 44 len 24576 12/30 Read block 252 len 32768 13/30 Write block 166 len 8192 data: 12 12 14/30 Read block 147 len 24576 15/30 Read block 348 len 12288 16/30 Read block 391 len 28672 17/30 Write block 513 len 20480 data: 9 4 6 6 8 18/30 Write block 92 len 16384 data: 14 15 16 13 19/30 Read block 31 len 12288 20/30 Read block 151 len 40960 21/30 Read block 170 len 20480 22/30 Read block 335 len 4096 23/30 Read block 508 len 40960 24/30 Read block 197 len 32768 25/30 Write block 593 len 24576 data: 8 6 9 8 7 6 26/30 Write block 109 len 28672 data: 10 8 8 8 9 9 8 27/30 Read block 517 len 24576 28/30 Write block 39 len 28672 data: 15 11 11 8 9 7 5 29/30 Write block 181 len 8192 data: 10 12 30/30 Read block 548 len 8192 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:57.763025924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 126 126 126 220 220 220 T T T 2 040-059 116 116 116 206 206 206 T T T 3 060-079 128 128 128 222 222 222 F F F 4 080-099 127 127 127 221 221 221 T T T 5 100-119 120 120 120 210 210 210 T T T 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 F F F 8 160-179 127 127 127 221 221 221 T T T 9 180-199 128 128 128 222 222 222 T T T 10 200-219 118 118 118 209 209 209 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 128 128 128 222 222 222 F F F 14 280-299 128 128 128 222 222 222 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 125 125 125 218 218 218 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 124 124 124 216 216 216 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 127 127 127 221 221 221 F F F 24 480-499 127 127 127 221 221 221 F F F 25 500-519 124 124 124 216 216 216 T T T 26 520-539 118 118 118 209 209 209 F F F 27 540-559 128 128 128 222 222 222 F F F 28 560-579 124 124 124 217 217 217 F F F 29 580-599 127 127 127 221 221 221 T T T Max gen: 128, Max flush: 222 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 129 129 129 223 223 223 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 128 128 128 222 222 222 F F F 4 080-099 129 129 129 223 223 223 F F F 5 100-119 129 129 129 223 223 223 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 F F F 8 160-179 129 129 129 223 223 223 F F F 9 180-199 129 129 129 223 223 223 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 124 124 124 215 215 215 F F F 12 240-259 120 120 120 210 210 210 F F F 13 260-279 128 128 128 222 222 222 F F F 14 280-299 128 128 128 222 222 222 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 114 114 114 201 201 201 F F F 19 380-399 125 125 125 218 218 218 F F F 20 400-419 126 126 126 219 219 219 F F F 21 420-439 124 124 124 216 216 216 F F F 22 440-459 126 126 126 219 219 219 F F F 23 460-479 127 127 127 221 221 221 F F F 24 480-499 127 127 127 221 221 221 F F F 25 500-519 129 129 129 223 223 223 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 128 128 128 222 222 222 F F F 28 560-579 124 124 124 217 217 217 F F F 29 580-599 129 129 129 223 223 223 F F F Max gen: 129, Max flush: 223 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.01804171Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.061245066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.061794052Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.061838242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.061857902Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.061873081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} {"msg":"Crucible 8573a1d1-e3e9-4151-b1a1-53ea37c93835 has session id: 6b417faf-9627-4599-8954-4af0ab0e5116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.062438567Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Upstairs opts: Upstairs UUID: 8573a1d1-e3e9-4151-b1a1-53ea37c93835, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.062478437Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Crucible stats registered with UUID: 8573a1d1-e3e9-4151-b1a1-53ea37c93835","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.062502687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:12.062524717Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.062981243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063014523Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} {"msg":"8573a1d1-e3e9-4151-b1a1-53ea37c93835 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063091412Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063118712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"0","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063157062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"0","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063184962Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"1","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063213342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"1","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063238831Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063263541Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063286061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063308291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063329521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"ds_connection connected from Ok(127.0.0.1:34398)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.06336616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"ds_connection connected from Ok(127.0.0.1:48763)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.06339361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:12.06342567Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:12.06345793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:12.063503149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063606059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.063918016Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"0","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.064068015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"1","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.064425362Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"0","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.064530392Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"1","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.065039678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.065063908Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.065208184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"ds_connection connected from Ok(127.0.0.1:61814)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.065545472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.066824432Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.067379668Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[0]R flush_numbers[0..12]: [218, 223, 223, 222, 223, 223, 210, 214, 223, 223, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.067993073Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[0]R generation[0..12]: [125, 129, 129, 128, 129, 129, 120, 123, 129, 129, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068017563Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068033383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[1]R flush_numbers[0..12]: [218, 223, 223, 222, 223, 223, 210, 214, 223, 223, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068047733Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[1]R generation[0..12]: [125, 129, 129, 128, 129, 129, 120, 123, 129, 129, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068061633Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068075613Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[2]R flush_numbers[0..12]: [218, 223, 223, 222, 223, 223, 210, 214, 223, 223, 209, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068089193Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[2]R generation[0..12]: [125, 129, 129, 128, 129, 129, 120, 123, 129, 129, 118, 124]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068107723Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068121943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068134912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068147932Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068160732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068173712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"":"downstairs","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068186772Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"Wait for a query_work_queue command to finish before sending IO 6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068208222Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"0","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068221392Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"1","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068234322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"client":"2","":"downstairs","session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068247391Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068259542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"8573a1d1-e3e9-4151-b1a1-53ea37c93835 is now active with session: 6b417faf-9627-4599-8954-4af0ab0e5116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068273321Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068285131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712,"session_id":"6b417faf-9627-4599-8954-4af0ab0e5116"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068297221Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} {"msg":"Activated sub_volume 8573a1d1-e3e9-4151-b1a1-53ea37c93835","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:22.068308461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 32768 data: 6 8 10 10 11 7 7 7 02/30 Flush 03/30 Write block 586 len 36864 data: 11 11 11 11 10 8 9 9 7 04/30 Read block 129 len 24576 05/30 Read block 431 len 12288 06/30 Write block 234 len 24576 data: 7 10 9 8 8 8 07/30 Read block 387 len 20480 08/30 Write block 66 len 40960 data: 12 12 12 13 11 11 13 13 11 14 09/30 Flush 10/30 Write block 396 len 28672 data: 5 5 5 6 9 10 11 11/30 Write block 424 len 12288 data: 11 11 10 12/30 Write block 273 len 12288 data: 11 9 11 13/30 Read block 47 len 32768 14/30 Read block 414 len 4096 15/30 Read block 475 len 4096 16/30 Write block 149 len 24576 data: 10 11 12 13 10 8 17/30 Read block 340 len 36864 18/30 Read block 295 len 32768 19/30 Write block 363 len 28672 data: 6 7 7 6 8 8 7 20/30 Read block 119 len 12288 21/30 Write block 252 len 16384 data: 8 9 13 11 22/30 Write block 585 len 16384 data: 11 12 12 12 23/30 Write block 153 len 16384 data: 11 9 7 8 24/30 Write block 576 len 16384 data: 11 11 12 12 25/30 Read block 484 len 32768 26/30 Read block 553 len 36864 27/30 Read block 406 len 32768 28/30 Read block 539 len 40960 29/30 Write block 265 len 4096 data: 11 30/30 Write block 486 len 24576 data: 15 13 11 14 14 15 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 7 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:22.089408493Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 129 129 129 223 223 223 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 129 129 129 223 223 223 F F F 5 100-119 129 129 129 223 223 223 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 123 123 123 214 214 214 T T T 8 160-179 129 129 129 223 223 223 F F F 9 180-199 129 129 129 223 223 223 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 130 130 130 225 225 225 F F F 12 240-259 120 120 120 210 210 210 T T T 13 260-279 128 128 128 222 222 222 T T T 14 280-299 128 128 128 222 222 222 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 114 114 114 201 201 201 T T T 19 380-399 125 125 125 218 218 218 T T T 20 400-419 126 126 126 219 219 219 T T T 21 420-439 124 124 124 216 216 216 T T T 22 440-459 130 130 130 224 224 224 F F F 23 460-479 130 130 130 224 224 224 F F F 24 480-499 127 127 127 221 221 221 T T T 25 500-519 129 129 129 223 223 223 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 128 128 128 222 222 222 F F F 28 560-579 124 124 124 217 217 217 T T T 29 580-599 130 130 130 225 225 225 T T T Max gen: 130, Max flush: 225 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 129 129 129 223 223 223 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 129 129 129 223 223 223 F F F 5 100-119 129 129 129 223 223 223 F F F 6 120-139 120 120 120 210 210 210 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 129 129 129 223 223 223 F F F 9 180-199 129 129 129 223 223 223 F F F 10 200-219 118 118 118 209 209 209 F F F 11 220-239 130 130 130 225 225 225 F F F 12 240-259 131 131 131 226 226 226 F F F 13 260-279 131 131 131 226 226 226 F F F 14 280-299 128 128 128 222 222 222 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 126 126 126 220 220 220 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 131 131 131 226 226 226 F F F 19 380-399 131 131 131 226 226 226 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 131 131 131 226 226 226 F F F 22 440-459 130 130 130 224 224 224 F F F 23 460-479 130 130 130 224 224 224 F F F 24 480-499 131 131 131 226 226 226 F F F 25 500-519 129 129 129 223 223 223 F F F 26 520-539 118 118 118 209 209 209 F F F 27 540-559 128 128 128 222 222 222 F F F 28 560-579 131 131 131 226 226 226 F F F 29 580-599 131 131 131 226 226 226 F F F Max gen: 131, Max flush: 226 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.350342505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:26.39254654Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.392895737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.423732806Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.424260482Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.424295542Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.424309942Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.424324291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} {"msg":"Crucible 5ca9ee01-9262-4f6e-897a-95edbed2ffc9 has session id: 19493ab8-3a08-4dad-af69-64fecea28d3b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.424874997Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Upstairs opts: Upstairs UUID: 5ca9ee01-9262-4f6e-897a-95edbed2ffc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.424904327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Crucible stats registered with UUID: 5ca9ee01-9262-4f6e-897a-95edbed2ffc9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.424919387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:26.424932237Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425377334Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425401043Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} {"msg":"5ca9ee01-9262-4f6e-897a-95edbed2ffc9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425473343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425492063Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"0","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425527833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"0","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425554972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"1","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425588152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"1","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425627242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"2","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425659812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"2","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425687241Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425713871Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425740541Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39504)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.42579712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54913)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.42591551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49573)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.425952299Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.426379156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"1","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.426420346Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"0","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.426451995Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"2","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427067211Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"1","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427096341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"0","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427127481Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"2","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427710946Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427737736Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[0]R flush_numbers[0..12]: [218, 223, 223, 225, 223, 223, 210, 226, 223, 223, 209, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427764646Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[0]R generation[0..12]: [125, 129, 129, 130, 129, 129, 120, 131, 129, 129, 118, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427780236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427795856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[1]R flush_numbers[0..12]: [218, 223, 223, 225, 223, 223, 210, 226, 223, 223, 209, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427810185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[1]R generation[0..12]: [125, 129, 129, 130, 129, 129, 120, 131, 129, 129, 118, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427824345Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427838275Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[2]R flush_numbers[0..12]: [218, 223, 223, 225, 223, 223, 210, 226, 223, 223, 209, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427851765Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[2]R generation[0..12]: [125, 129, 129, 130, 129, 129, 120, 131, 129, 129, 118, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427865205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427878825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427891365Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427904385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427920675Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427933815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"":"downstairs","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427946964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427968004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"0","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.427990944Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"1","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.428014094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"client":"2","":"downstairs","session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.428027554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.428039884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"5ca9ee01-9262-4f6e-897a-95edbed2ffc9 is now active with session: 19493ab8-3a08-4dad-af69-64fecea28d3b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.428052594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.428064414Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720,"session_id":"19493ab8-3a08-4dad-af69-64fecea28d3b"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.428076723Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} {"msg":"Activated sub_volume 5ca9ee01-9262-4f6e-897a-95edbed2ffc9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:26.428087783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 385 len 4096 data: 15 02/30 Write block 374 len 24576 data: 10 10 9 8 11 8 03/30 Read block 128 len 16384 04/30 Write block 499 len 12288 data: 11 10 10 05/30 Read block 487 len 32768 06/30 Read block 117 len 32768 07/30 Read block 262 len 36864 08/30 Read block 502 len 4096 09/30 Write block 427 len 40960 data: 10 9 12 13 17 14 14 14 14 11 10/30 Write block 330 len 40960 data: 11 9 9 7 9 10 11 13 14 12 11/30 Read block 319 len 28672 12/30 Write block 83 len 20480 data: 7 9 8 8 10 13/30 Read block 197 len 40960 14/30 Write block 398 len 4096 data: 6 15/30 Write block 282 len 32768 data: 9 8 10 13 12 14 14 13 16/30 Read block 382 len 12288 17/30 Write block 375 len 40960 data: 11 10 9 12 9 8 8 8 9 12 18/30 Read block 261 len 12288 19/30 Write block 122 len 16384 data: 11 13 15 14 20/30 Read block 534 len 12288 21/30 Write block 515 len 32768 data: 7 7 9 10 13 13 16 16 22/30 Write block 89 len 8192 data: 11 14 23/30 Flush 24/30 Write block 289 len 28672 data: 14 14 15 15 16 11 8 25/30 Read block 171 len 20480 26/30 Write block 555 len 24576 data: 15 17 18 15 13 14 27/30 Read block 544 len 16384 28/30 Write block 34 len 24576 data: 13 14 15 14 16 16 29/30 Read block 144 len 8192 30/30 Write block 198 len 24576 data: 6 7 9 9 8 7 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:26.449422074Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 129 129 129 223 223 223 T T T 2 040-059 129 129 129 223 223 223 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 132 132 132 227 227 227 F F F 5 100-119 129 129 129 223 223 223 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 129 129 129 223 223 223 F F F 9 180-199 129 129 129 223 223 223 T T T 10 200-219 118 118 118 209 209 209 T T T 11 220-239 130 130 130 225 225 225 F F F 12 240-259 131 131 131 226 226 226 F F F 13 260-279 131 131 131 226 226 226 F F F 14 280-299 132 132 132 227 227 227 T T T 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 132 132 132 227 227 227 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 132 132 132 227 227 227 F F F 22 440-459 130 130 130 224 224 224 F F F 23 460-479 130 130 130 224 224 224 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 F F F 26 520-539 132 132 132 227 227 227 F F F 27 540-559 128 128 128 222 222 222 T T T 28 560-579 131 131 131 226 226 226 T T T 29 580-599 131 131 131 226 226 226 F F F Max gen: 132, Max flush: 227 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 133 133 133 228 228 228 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 130 130 130 225 225 225 F F F 4 080-099 132 132 132 227 227 227 F F F 5 100-119 129 129 129 223 223 223 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 129 129 129 223 223 223 F F F 9 180-199 133 133 133 228 228 228 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 130 130 130 225 225 225 F F F 12 240-259 131 131 131 226 226 226 F F F 13 260-279 131 131 131 226 226 226 F F F 14 280-299 133 133 133 228 228 228 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 132 132 132 227 227 227 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 132 132 132 227 227 227 F F F 22 440-459 130 130 130 224 224 224 F F F 23 460-479 130 130 130 224 224 224 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 F F F 26 520-539 132 132 132 227 227 227 F F F 27 540-559 133 133 133 228 228 228 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 131 131 131 226 226 226 F F F Max gen: 133, Max flush: 228 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.72064424Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.764286724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.764891699Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.764947519Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.764967089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.764986028Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} {"msg":"Crucible 863762aa-4a64-47e5-8dcd-dc22c65a8876 has session id: a59e1684-fb3d-4b09-9de0-edf9a060a407","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765504035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Upstairs opts: Upstairs UUID: 863762aa-4a64-47e5-8dcd-dc22c65a8876, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765543564Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Crucible stats registered with UUID: 863762aa-4a64-47e5-8dcd-dc22c65a8876","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765566974Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:30.765592434Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766024121Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766058951Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} {"msg":"863762aa-4a64-47e5-8dcd-dc22c65a8876 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76612851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76615727Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"0","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766198979Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"0","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766267449Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"1","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766302259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"1","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766330018Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"2","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766356428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"2","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766382858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766409518Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766436128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"ds_connection connected from Ok(127.0.0.1:60396)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766463517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"ds_connection connected from Ok(127.0.0.1:40863)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766623606Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"ds_connection connected from Ok(127.0.0.1:49185)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766676326Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.767226172Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"0","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.767262622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"1","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.767279581Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"2","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768031866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"0","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768072785Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"1","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768103965Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"2","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768653831Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768679881Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[0]R flush_numbers[0..12]: [218, 228, 223, 225, 227, 223, 227, 226, 223, 228, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768706131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[0]R generation[0..12]: [125, 133, 129, 130, 132, 129, 132, 131, 129, 133, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76872154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76873583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[1]R flush_numbers[0..12]: [218, 228, 223, 225, 227, 223, 227, 226, 223, 228, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76874961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[1]R generation[0..12]: [125, 133, 129, 130, 132, 129, 132, 131, 129, 133, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76876358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76878189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[2]R flush_numbers[0..12]: [218, 228, 223, 225, 227, 223, 227, 226, 223, 228, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76879604Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[2]R generation[0..12]: [125, 133, 129, 130, 132, 129, 132, 131, 129, 133, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.7688098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76882388Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76883671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768849799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76886269Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768875469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"":"downstairs","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:30.768888419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768905549Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"0","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768918679Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"1","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768931659Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"client":"2","":"downstairs","session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768944529Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768956539Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"863762aa-4a64-47e5-8dcd-dc22c65a8876 is now active with session: a59e1684-fb3d-4b09-9de0-edf9a060a407","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768969229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768983189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728,"session_id":"a59e1684-fb3d-4b09-9de0-edf9a060a407"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768995438Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} {"msg":"Activated sub_volume 863762aa-4a64-47e5-8dcd-dc22c65a8876","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769006498Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 31 len 8192 data: 10 10 03/30 Write block 297 len 4096 data: 6 04/30 Write block 172 len 24576 data: 8 10 10 10 10 9 05/30 Read block 476 len 4096 06/30 Write block 141 len 12288 data: 10 11 11 07/30 Read block 147 len 40960 08/30 Flush 09/30 Write block 594 len 16384 data: 8 10 9 8 10/30 Read block 313 len 40960 11/30 Write block 73 len 12288 data: 14 12 15 12/30 Read block 538 len 24576 13/30 Write block 379 len 12288 data: 10 9 9 14/30 Read block 270 len 24576 15/30 Read block 76 len 4096 16/30 Flush 17/30 Flush 18/30 Write block 388 len 12288 data: 12 14 12 19/30 Read block 208 len 40960 20/30 Write block 112 len 16384 data: 9 10 10 9 21/30 Flush 22/30 Write block 511 len 16384 data: 12 9 10 5 23/30 Read block 560 len 40960 24/30 Read block 508 len 16384 25/30 Write block 192 len 20480 data: 8 7 7 9 9 26/30 Read block 561 len 8192 27/30 Read block 518 len 20480 28/30 Read block 22 len 20480 29/30 Read block 519 len 28672 30/30 Write block 538 len 16384 data: 6 8 8 10 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:30.788602802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 134 134 134 230 230 230 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 132 132 132 227 227 227 F F F 5 100-119 134 134 134 233 233 233 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 134 134 134 230 230 230 F F F 8 160-179 134 134 134 230 230 230 F F F 9 180-199 133 133 133 228 228 228 T T T 10 200-219 133 133 133 228 228 228 F F F 11 220-239 130 130 130 225 225 225 F F F 12 240-259 131 131 131 226 226 226 F F F 13 260-279 131 131 131 226 226 226 F F F 14 280-299 134 134 134 230 230 230 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 134 134 134 231 231 231 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 132 132 132 227 227 227 F F F 22 440-459 130 130 130 224 224 224 F F F 23 460-479 130 130 130 224 224 224 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 T T T 26 520-539 132 132 132 227 227 227 T T T 27 540-559 133 133 133 228 228 228 T T T 28 560-579 133 133 133 228 228 228 F F F 29 580-599 134 134 134 231 231 231 F F F Max gen: 134, Max flush: 233 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 125 125 125 218 218 218 F F F 1 020-039 134 134 134 230 230 230 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 132 132 132 227 227 227 F F F 5 100-119 134 134 134 233 233 233 F F F 6 120-139 132 132 132 227 227 227 F F F 7 140-159 134 134 134 230 230 230 F F F 8 160-179 134 134 134 230 230 230 F F F 9 180-199 135 135 135 234 234 234 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 130 130 130 225 225 225 F F F 12 240-259 131 131 131 226 226 226 F F F 13 260-279 131 131 131 226 226 226 F F F 14 280-299 134 134 134 230 230 230 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 134 134 134 231 231 231 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 132 132 132 227 227 227 F F F 22 440-459 130 130 130 224 224 224 F F F 23 460-479 130 130 130 224 224 224 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 135 135 135 234 234 234 F F F 26 520-539 135 135 135 234 234 234 F F F 27 540-559 135 135 135 234 234 234 F F F 28 560-579 133 133 133 228 228 228 F F F 29 580-599 134 134 134 231 231 231 F F F Max gen: 135, Max flush: 234 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.022632146Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:35.065078838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.065455216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.096288035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09684355Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09688558Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09690108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09691676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} {"msg":"Crucible e031b636-9460-4643-a348-ae43bc9eb0ea has session id: 239baa3f-593c-4057-9226-25320d992a14","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.097508945Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Upstairs opts: Upstairs UUID: e031b636-9460-4643-a348-ae43bc9eb0ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.097549975Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Crucible stats registered with UUID: e031b636-9460-4643-a348-ae43bc9eb0ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.097573875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:35.097594925Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098027312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098061211Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} {"msg":"e031b636-9460-4643-a348-ae43bc9eb0ea active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098122351Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098149301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"0","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098177211Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"0","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09819149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"1","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.0982059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"1","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09822598Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"2","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09825264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"2","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09826888Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.09829115Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098318049Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"ds_connection connected from Ok(127.0.0.1:48868)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098344149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"ds_connection connected from Ok(127.0.0.1:42145)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098586397Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"ds_connection connected from Ok(127.0.0.1:52365)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.098629787Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.099019364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"1","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.099147023Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"2","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.099271492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"0","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.099684739Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"2","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.099812488Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"1","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.099836688Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"0","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100253365Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100382124Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[0]R flush_numbers[0..12]: [218, 230, 223, 231, 227, 233, 227, 230, 230, 234, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100500883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[0]R generation[0..12]: [125, 134, 129, 134, 132, 134, 132, 134, 134, 135, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100522063Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100537293Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[1]R flush_numbers[0..12]: [218, 230, 223, 231, 227, 233, 227, 230, 230, 234, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100551253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[1]R generation[0..12]: [125, 134, 129, 134, 132, 134, 132, 134, 134, 135, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100565482Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100579362Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[2]R flush_numbers[0..12]: [218, 230, 223, 231, 227, 233, 227, 230, 230, 234, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100593052Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[2]R generation[0..12]: [125, 134, 129, 134, 132, 134, 132, 134, 134, 135, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100606532Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100619992Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100633482Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100657552Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100688762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100716941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"":"downstairs","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100740531Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100754991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"0","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100778191Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"1","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100802621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"client":"2","":"downstairs","session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.100825711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.10084712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"e031b636-9460-4643-a348-ae43bc9eb0ea is now active with session: 239baa3f-593c-4057-9226-25320d992a14","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.10087233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.10089493Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736,"session_id":"239baa3f-593c-4057-9226-25320d992a14"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.1009094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} {"msg":"Activated sub_volume e031b636-9460-4643-a348-ae43bc9eb0ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.10092075Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 22 len 12288 data: 15 19 16 03/30 Read block 138 len 40960 04/30 Write block 461 len 16384 data: 8 11 12 10 05/30 Read block 264 len 12288 06/30 Read block 11 len 12288 07/30 Read block 157 len 4096 08/30 Read block 78 len 40960 09/30 Write block 36 len 4096 data: 16 10/30 Write block 27 len 40960 data: 12 10 6 7 11 11 12 14 15 17 11/30 Write block 145 len 20480 data: 9 8 8 10 11 12/30 Read block 154 len 12288 13/30 Write block 447 len 4096 data: 8 14/30 Flush 15/30 Write block 482 len 32768 data: 12 14 14 14 16 14 12 15 16/30 Read block 469 len 24576 17/30 Read block 161 len 36864 18/30 Write block 401 len 20480 data: 11 12 14 14 13 19/30 Read block 404 len 20480 20/30 Read block 68 len 8192 21/30 Write block 5 len 36864 data: 10 12 14 12 14 14 16 14 13 22/30 Write block 119 len 24576 data: 12 12 11 12 14 16 23/30 Read block 518 len 8192 24/30 Write block 478 len 24576 data: 6 9 7 7 13 15 25/30 Flush 26/30 Write block 240 len 12288 data: 7 8 7 27/30 Read block 584 len 36864 28/30 Read block 35 len 20480 29/30 Read block 80 len 24576 30/30 Write block 574 len 40960 data: 7 9 12 12 13 13 14 11 12 11 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:35.121276058Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 136 136 136 236 236 236 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 132 132 132 227 227 227 F F F 5 100-119 136 136 136 237 237 237 F F F 6 120-139 136 136 136 237 237 237 F F F 7 140-159 136 136 136 236 236 236 F F F 8 160-179 134 134 134 230 230 230 F F F 9 180-199 135 135 135 234 234 234 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 130 130 130 225 225 225 F F F 12 240-259 131 131 131 226 226 226 T T T 13 260-279 131 131 131 226 226 226 F F F 14 280-299 134 134 134 230 230 230 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 134 134 134 231 231 231 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 136 136 136 237 237 237 F F F 21 420-439 132 132 132 227 227 227 F F F 22 440-459 136 136 136 236 236 236 F F F 23 460-479 136 136 136 237 237 237 F F F 24 480-499 136 136 136 237 237 237 F F F 25 500-519 135 135 135 234 234 234 F F F 26 520-539 135 135 135 234 234 234 F F F 27 540-559 135 135 135 234 234 234 F F F 28 560-579 133 133 133 228 228 228 T T T 29 580-599 134 134 134 231 231 231 T T T Max gen: 136, Max flush: 237 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 136 136 136 236 236 236 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 132 132 132 227 227 227 F F F 5 100-119 136 136 136 237 237 237 F F F 6 120-139 136 136 136 237 237 237 F F F 7 140-159 136 136 136 236 236 236 F F F 8 160-179 134 134 134 230 230 230 F F F 9 180-199 135 135 135 234 234 234 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 130 130 130 225 225 225 F F F 12 240-259 137 137 137 238 238 238 F F F 13 260-279 131 131 131 226 226 226 F F F 14 280-299 134 134 134 230 230 230 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 134 134 134 231 231 231 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 136 136 136 237 237 237 F F F 21 420-439 132 132 132 227 227 227 F F F 22 440-459 136 136 136 236 236 236 F F F 23 460-479 136 136 136 237 237 237 F F F 24 480-499 136 136 136 237 237 237 F F F 25 500-519 135 135 135 234 234 234 F F F 26 520-539 135 135 135 234 234 234 F F F 27 540-559 135 135 135 234 234 234 F F F 28 560-579 137 137 137 238 238 238 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 137, Max flush: 238 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.484198277Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.529337419Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.529925485Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.529968714Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.529984484Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.530000004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} {"msg":"Crucible 00ee9078-a70a-47c3-a96e-91046977af6a has session id: c0c57dcb-b3f1-42db-ac0e-9dd093c72960","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53049725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Upstairs opts: Upstairs UUID: 00ee9078-a70a-47c3-a96e-91046977af6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.530637869Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Crucible stats registered with UUID: 00ee9078-a70a-47c3-a96e-91046977af6a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.530669089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:39.530684439Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531060456Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531087776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} {"msg":"00ee9078-a70a-47c3-a96e-91046977af6a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531150185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531167755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"0","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531196295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"0","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531210535Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"1","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531224555Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"1","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531244495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"2","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531300864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"2","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531338674Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531359364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531385534Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"ds_connection connected from Ok(127.0.0.1:47937)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531413454Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"ds_connection connected from Ok(127.0.0.1:51229)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531437943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"ds_connection connected from Ok(127.0.0.1:41230)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.531464863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.532074929Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"1","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.532107668Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"0","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.532802603Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"0","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.532832753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"1","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.5332678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"2","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.533475898Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.533580087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.533850555Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"2","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[0]R flush_numbers[0..12]: [237, 236, 223, 231, 227, 237, 237, 236, 230, 234, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534361681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[0]R generation[0..12]: [136, 136, 129, 134, 132, 136, 136, 136, 134, 135, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534383301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534398151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[1]R flush_numbers[0..12]: [237, 236, 223, 231, 227, 237, 237, 236, 230, 234, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534412001Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[1]R generation[0..12]: [136, 136, 129, 134, 132, 136, 136, 136, 134, 135, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534425561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534439381Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[2]R flush_numbers[0..12]: [237, 236, 223, 231, 227, 237, 237, 236, 230, 234, 228, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534453081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[2]R generation[0..12]: [136, 136, 129, 134, 132, 136, 136, 136, 134, 135, 133, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534469971Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53448418Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53449688Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53450988Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.5345229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.5345358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"":"downstairs","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53454862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53456042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"0","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53457338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"1","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:39.53458612Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"client":"2","":"downstairs","session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534605839Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.53461803Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"00ee9078-a70a-47c3-a96e-91046977af6a is now active with session: c0c57dcb-b3f1-42db-ac0e-9dd093c72960","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534630869Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534642619Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744,"session_id":"c0c57dcb-b3f1-42db-ac0e-9dd093c72960"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534656009Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} {"msg":"Activated sub_volume 00ee9078-a70a-47c3-a96e-91046977af6a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.534667579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 417 len 28672 data: 6 6 8 7 10 10 11 02/30 Write block 136 len 20480 data: 13 12 11 10 12 03/30 Flush 04/30 Write block 269 len 20480 data: 10 12 12 12 12 05/30 Write block 234 len 24576 data: 8 11 10 9 9 9 06/30 Write block 97 len 24576 data: 9 7 9 10 7 5 07/30 Write block 240 len 32768 data: 8 9 8 8 8 7 7 8 08/30 Read block 309 len 16384 09/30 Read block 31 len 28672 10/30 Read block 426 len 4096 11/30 Write block 196 len 8192 data: 10 8 12/30 Read block 456 len 24576 13/30 Write block 265 len 4096 data: 12 14/30 Write block 364 len 4096 data: 8 15/30 Read block 126 len 4096 16/30 Read block 533 len 24576 17/30 Write block 135 len 32768 data: 12 14 13 12 11 13 11 12 18/30 Read block 534 len 24576 19/30 Write block 148 len 36864 data: 11 12 12 13 14 12 10 8 9 20/30 Read block 230 len 40960 21/30 Read block 90 len 16384 22/30 Read block 177 len 16384 23/30 Read block 187 len 28672 24/30 Write block 370 len 8192 data: 8 7 25/30 Read block 349 len 24576 26/30 Flush 27/30 Write block 468 len 20480 data: 9 9 11 9 7 28/30 Write block 176 len 36864 data: 11 10 9 11 11 11 13 12 14 29/30 Read block 579 len 16384 30/30 Read block 214 len 8192 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:39.554395451Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 136 136 136 236 236 236 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 138 138 138 240 240 240 F F F 5 100-119 138 138 138 240 240 240 F F F 6 120-139 138 138 138 240 240 240 F F F 7 140-159 138 138 138 240 240 240 F F F 8 160-179 134 134 134 230 230 230 T T T 9 180-199 138 138 138 240 240 240 T T T 10 200-219 133 133 133 228 228 228 F F F 11 220-239 138 138 138 240 240 240 F F F 12 240-259 138 138 138 240 240 240 F F F 13 260-279 138 138 138 240 240 240 F F F 14 280-299 134 134 134 230 230 230 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 138 138 138 239 239 239 F F F 22 440-459 136 136 136 236 236 236 F F F 23 460-479 136 136 136 237 237 237 T T T 24 480-499 136 136 136 237 237 237 F F F 25 500-519 135 135 135 234 234 234 F F F 26 520-539 135 135 135 234 234 234 F F F 27 540-559 135 135 135 234 234 234 F F F 28 560-579 137 137 137 238 238 238 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 138, Max flush: 240 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 136 136 136 236 236 236 F F F 2 040-059 129 129 129 223 223 223 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 138 138 138 240 240 240 F F F 5 100-119 138 138 138 240 240 240 F F F 6 120-139 138 138 138 240 240 240 F F F 7 140-159 138 138 138 240 240 240 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 139 139 139 241 241 241 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 138 138 138 240 240 240 F F F 12 240-259 138 138 138 240 240 240 F F F 13 260-279 138 138 138 240 240 240 F F F 14 280-299 134 134 134 230 230 230 F F F 15 300-319 125 125 125 218 218 218 F F F 16 320-339 132 132 132 227 227 227 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 138 138 138 239 239 239 F F F 22 440-459 136 136 136 236 236 236 F F F 23 460-479 139 139 139 241 241 241 F F F 24 480-499 136 136 136 237 237 237 F F F 25 500-519 135 135 135 234 234 234 F F F 26 520-539 135 135 135 234 234 234 F F F 27 540-559 135 135 135 234 234 234 F F F 28 560-579 137 137 137 238 238 238 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 139, Max flush: 241 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.780100958Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.824435916Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.825034302Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.825091371Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.825110631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.825128841Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} {"msg":"Crucible 0fb8df71-9bb3-4af4-807b-527f1fd4e528 has session id: 3af8f543-5d7f-4bb7-8688-49945e4d5db3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.825725557Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Upstairs opts: Upstairs UUID: 0fb8df71-9bb3-4af4-807b-527f1fd4e528, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.825763886Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Crucible stats registered with UUID: 0fb8df71-9bb3-4af4-807b-527f1fd4e528","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.825782316Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:43.825795526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826227683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826250443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} {"msg":"0fb8df71-9bb3-4af4-807b-527f1fd4e528 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826323892Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826342822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"0","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826369782Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"0","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826383632Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826397732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826410592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"2","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826432781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"2","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826455631Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826479321Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826506561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:61202)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.82657708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56069)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826743349Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:43.826860958Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:43.826902608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:43.826934648Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.826989287Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.827175976Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"0","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.827210346Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"2","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.827848151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"0","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.827888621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"2","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.828451266Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.828473856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.828445074Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41520)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.828525483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.829537216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830050822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[0]R flush_numbers[0..12]: [237, 236, 223, 231, 240, 240, 240, 240, 241, 241, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830659098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[0]R generation[0..12]: [136, 136, 129, 134, 138, 138, 138, 138, 139, 139, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830683438Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830702047Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[1]R flush_numbers[0..12]: [237, 236, 223, 231, 240, 240, 240, 240, 241, 241, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830719407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[1]R generation[0..12]: [136, 136, 129, 134, 138, 138, 138, 138, 139, 139, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830736697Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830753967Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[2]R flush_numbers[0..12]: [237, 236, 223, 231, 240, 240, 240, 240, 241, 241, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830770717Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[2]R generation[0..12]: [136, 136, 129, 134, 138, 138, 138, 138, 139, 139, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830787657Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830804766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830820576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830836776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830857086Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830873626Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"":"downstairs","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830889776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830904686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"0","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830920876Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"1","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830936845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"client":"2","":"downstairs","session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830953156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830968425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"Wait for a query_work_queue command to finish before sending IO 3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"0fb8df71-9bb3-4af4-807b-527f1fd4e528 is now active with session: 3af8f543-5d7f-4bb7-8688-49945e4d5db3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.830992725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.831007585Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752,"session_id":"3af8f543-5d7f-4bb7-8688-49945e4d5db3"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.831022665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} {"msg":"Activated sub_volume 0fb8df71-9bb3-4af4-807b-527f1fd4e528","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.831036245Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 181 len 28672 data: 12 14 13 15 13 12 11 02/30 Read block 301 len 20480 03/30 Read block 306 len 24576 04/30 Write block 275 len 40960 data: 12 7 5 8 8 9 10 10 9 11 05/30 Read block 335 len 32768 06/30 Write block 420 len 40960 data: 8 11 11 12 12 12 11 11 10 13 07/30 Read block 551 len 24576 08/30 Write block 169 len 20480 data: 11 11 8 9 11 09/30 Read block 582 len 28672 10/30 Read block 288 len 8192 11/30 Read block 69 len 20480 12/30 Read block 290 len 36864 13/30 Write block 127 len 36864 data: 11 11 12 13 17 15 14 13 13 14/30 Write block 484 len 20480 data: 15 15 17 15 13 15/30 Read block 14 len 8192 16/30 Read block 502 len 4096 17/30 Read block 458 len 12288 18/30 Write block 496 len 4096 data: 9 19/30 Write block 292 len 40960 data: 16 17 12 9 6 7 6 9 9 9 20/30 Write block 131 len 20480 data: 18 16 15 14 14 21/30 Write block 587 len 4096 data: 13 22/30 Write block 509 len 8192 data: 12 12 23/30 Write block 524 len 28672 data: 11 10 12 11 8 9 7 24/30 Write block 249 len 8192 data: 12 12 25/30 Read block 275 len 12288 26/30 Read block 560 len 8192 27/30 Write block 516 len 20480 data: 8 10 11 14 14 28/30 Write block 35 len 32768 data: 16 18 15 17 17 12 12 9 29/30 Read block 502 len 32768 30/30 Write block 332 len 8192 data: 10 8 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:53.851172284Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 136 136 136 236 236 236 T T T 2 040-059 129 129 129 223 223 223 T T T 3 060-079 134 134 134 231 231 231 F F F 4 080-099 138 138 138 240 240 240 F F F 5 100-119 138 138 138 240 240 240 F F F 6 120-139 138 138 138 240 240 240 T T T 7 140-159 138 138 138 240 240 240 F F F 8 160-179 139 139 139 241 241 241 T T T 9 180-199 139 139 139 241 241 241 T T T 10 200-219 133 133 133 228 228 228 F F F 11 220-239 138 138 138 240 240 240 F F F 12 240-259 138 138 138 240 240 240 T T T 13 260-279 138 138 138 240 240 240 T T T 14 280-299 134 134 134 230 230 230 T T T 15 300-319 125 125 125 218 218 218 T T T 16 320-339 132 132 132 227 227 227 T T T 17 340-359 125 125 125 218 218 218 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 138 138 138 239 239 239 T T T 22 440-459 136 136 136 236 236 236 F F F 23 460-479 139 139 139 241 241 241 F F F 24 480-499 136 136 136 237 237 237 T T T 25 500-519 135 135 135 234 234 234 T T T 26 520-539 135 135 135 234 234 234 T T T 27 540-559 135 135 135 234 234 234 F F F 28 560-579 137 137 137 238 238 238 F F F 29 580-599 137 137 137 238 238 238 T T T Max gen: 139, Max flush: 241 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 140 140 140 242 242 242 F F F 2 040-059 140 140 140 242 242 242 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 138 138 138 240 240 240 F F F 5 100-119 138 138 138 240 240 240 F F F 6 120-139 140 140 140 242 242 242 F F F 7 140-159 138 138 138 240 240 240 F F F 8 160-179 140 140 140 242 242 242 F F F 9 180-199 140 140 140 242 242 242 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 138 138 138 240 240 240 F F F 12 240-259 140 140 140 242 242 242 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 140 140 140 242 242 242 F F F 15 300-319 140 140 140 242 242 242 F F F 16 320-339 140 140 140 242 242 242 F F F 17 340-359 125 125 125 218 218 218 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 134 134 134 233 233 233 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 140 140 140 242 242 242 F F F 22 440-459 136 136 136 236 236 236 F F F 23 460-479 139 139 139 241 241 241 F F F 24 480-499 140 140 140 242 242 242 F F F 25 500-519 140 140 140 242 242 242 F F F 26 520-539 140 140 140 242 242 242 F F F 27 540-559 135 135 135 234 234 234 F F F 28 560-579 137 137 137 238 238 238 F F F 29 580-599 140 140 140 242 242 242 F F F Max gen: 140, Max flush: 242 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.143172445Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.18791081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.188518495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.188568185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.188585865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.188609014Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} {"msg":"Crucible 175bc955-3d5d-4a87-80a8-dfee400897fb has session id: e4cd496c-8b65-44c8-bf1a-abccad1de79f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189122051Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Upstairs opts: Upstairs UUID: 175bc955-3d5d-4a87-80a8-dfee400897fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.1891689Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Crucible stats registered with UUID: 175bc955-3d5d-4a87-80a8-dfee400897fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.18919142Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:58.18920765Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189792796Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189823585Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} {"msg":"175bc955-3d5d-4a87-80a8-dfee400897fb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189899985Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189927215Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"0","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189955495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"0","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189970064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"1","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.189984324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"1","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190078123Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"2","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190109973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"2","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190141433Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190169493Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190196453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"ds_connection connected from Ok(127.0.0.1:65527)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190224762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59625)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190297202Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47083)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190331452Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190822248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"1","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190853628Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"0","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.190981687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"2","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.191515533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"1","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.191664412Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"0","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.191693411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"2","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192116228Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192238327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[0]R flush_numbers[0..12]: [237, 242, 242, 231, 240, 240, 242, 240, 242, 242, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192364316Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[0]R generation[0..12]: [136, 140, 140, 134, 138, 138, 140, 138, 140, 140, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192386936Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192401896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[1]R flush_numbers[0..12]: [237, 242, 242, 231, 240, 240, 242, 240, 242, 242, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192415616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[1]R generation[0..12]: [136, 140, 140, 134, 138, 138, 140, 138, 140, 140, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192429206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192442946Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[2]R flush_numbers[0..12]: [237, 242, 242, 231, 240, 240, 242, 240, 242, 242, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192456426Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[2]R generation[0..12]: [136, 140, 140, 134, 138, 138, 140, 138, 140, 140, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192470045Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192483756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192496455Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192509425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192522205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192596145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"":"downstairs","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192614184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192626615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"0","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192642614Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"1","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192655794Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"client":"2","":"downstairs","session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192668834Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192680914Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"175bc955-3d5d-4a87-80a8-dfee400897fb is now active with session: e4cd496c-8b65-44c8-bf1a-abccad1de79f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192693474Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192705304Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760,"session_id":"e4cd496c-8b65-44c8-bf1a-abccad1de79f"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192717624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} {"msg":"Activated sub_volume 175bc955-3d5d-4a87-80a8-dfee400897fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:58.192728794Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 537 len 28672 data: 5 7 9 9 11 10 9 02/30 Write block 390 len 16384 data: 13 10 8 7 03/30 Read block 436 len 36864 04/30 Read block 379 len 24576 05/30 Read block 175 len 24576 06/30 Write block 186 len 40960 data: 13 12 8 9 10 10 9 8 8 10 07/30 Write block 125 len 40960 data: 15 10 12 12 13 14 19 17 16 15 08/30 Write block 150 len 40960 data: 13 14 15 13 11 9 10 8 11 10 09/30 Write block 540 len 32768 data: 10 12 11 10 13 15 15 19 10/30 Flush 11/30 Write block 597 len 4096 data: 9 12/30 Read block 147 len 24576 13/30 Read block 89 len 4096 14/30 Read block 192 len 40960 15/30 Write block 48 len 24576 data: 8 8 6 7 7 8 16/30 Read block 581 len 40960 17/30 Read block 402 len 20480 18/30 Read block 198 len 24576 19/30 Read block 80 len 12288 20/30 Flush 21/30 Write block 82 len 24576 data: 9 8 10 9 9 11 22/30 Write block 350 len 20480 data: 13 18 17 16 18 23/30 Write block 112 len 4096 data: 10 24/30 Write block 572 len 24576 data: 9 8 8 10 13 13 25/30 Write block 487 len 28672 data: 16 14 16 15 16 19 18 26/30 Write block 248 len 20480 data: 12 13 13 10 9 27/30 Read block 207 len 4096 28/30 Read block 82 len 4096 29/30 Read block 317 len 28672 30/30 Write block 37 len 36864 data: 16 18 18 13 13 10 10 8 6 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:58.212463896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 140 140 140 242 242 242 T T T 2 040-059 142 142 142 244 244 244 T T T 3 060-079 134 134 134 231 231 231 F F F 4 080-099 138 138 138 240 240 240 T T T 5 100-119 138 138 138 240 240 240 T T T 6 120-139 142 142 142 243 243 243 F F F 7 140-159 142 142 142 243 243 243 F F F 8 160-179 140 140 140 242 242 242 F F F 9 180-199 142 142 142 243 243 243 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 138 138 138 240 240 240 F F F 12 240-259 140 140 140 242 242 242 T T T 13 260-279 140 140 140 242 242 242 F F F 14 280-299 140 140 140 242 242 242 F F F 15 300-319 140 140 140 242 242 242 F F F 16 320-339 140 140 140 242 242 242 F F F 17 340-359 125 125 125 218 218 218 T T T 18 360-379 138 138 138 240 240 240 F F F 19 380-399 142 142 142 243 243 243 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 140 140 140 242 242 242 F F F 22 440-459 136 136 136 236 236 236 F F F 23 460-479 139 139 139 241 241 241 F F F 24 480-499 140 140 140 242 242 242 T T T 25 500-519 140 140 140 242 242 242 F F F 26 520-539 142 142 142 243 243 243 F F F 27 540-559 142 142 142 243 243 243 F F F 28 560-579 137 137 137 238 238 238 T T T 29 580-599 142 142 142 244 244 244 F F F Max gen: 142, Max flush: 244 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 F F F 1 020-039 143 143 143 245 245 245 F F F 2 040-059 143 143 143 245 245 245 F F F 3 060-079 134 134 134 231 231 231 F F F 4 080-099 143 143 143 245 245 245 F F F 5 100-119 143 143 143 245 245 245 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 142 142 142 243 243 243 F F F 8 160-179 140 140 140 242 242 242 F F F 9 180-199 142 142 142 243 243 243 F F F 10 200-219 133 133 133 228 228 228 F F F 11 220-239 138 138 138 240 240 240 F F F 12 240-259 143 143 143 245 245 245 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 140 140 140 242 242 242 F F F 15 300-319 140 140 140 242 242 242 F F F 16 320-339 140 140 140 242 242 242 F F F 17 340-359 143 143 143 245 245 245 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 142 142 142 243 243 243 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 140 140 140 242 242 242 F F F 22 440-459 136 136 136 236 236 236 F F F 23 460-479 139 139 139 241 241 241 F F F 24 480-499 143 143 143 245 245 245 F F F 25 500-519 140 140 140 242 242 242 F F F 26 520-539 142 142 142 243 243 243 F F F 27 540-559 142 142 142 243 243 243 F F F 28 560-579 143 143 143 245 245 245 F F F 29 580-599 142 142 142 244 244 244 F F F Max gen: 143, Max flush: 245 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.466365001Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.510128464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.510743269Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.510786799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.510801669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.510816309Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} {"msg":"Crucible bbb4303d-5e81-4677-9bff-13442bd365ac has session id: 8e53f39d-4690-4fa4-903e-1c33ab7f8605","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.511247176Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Upstairs opts: Upstairs UUID: bbb4303d-5e81-4677-9bff-13442bd365ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.511285845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Crucible stats registered with UUID: bbb4303d-5e81-4677-9bff-13442bd365ac","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.511312145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:02.511327385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.511825191Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.511849981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} {"msg":"bbb4303d-5e81-4677-9bff-13442bd365ac active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.511917051Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.51193588Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"0","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.51196753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"0","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.51199243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"1","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.51201573Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"1","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512036939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"2","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512075229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"2","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512104789Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512132669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512161169Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"ds_connection connected from Ok(127.0.0.1:33761)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512210008Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"ds_connection connected from Ok(127.0.0.1:63186)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512415527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"ds_connection connected from Ok(127.0.0.1:35870)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512455096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512930523Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"0","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512959483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"1","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.512980552Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"2","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.513568308Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"1","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.513599528Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"2","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.513627438Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"0","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514121984Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514225183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[0]R flush_numbers[0..12]: [237, 245, 245, 231, 245, 245, 243, 243, 242, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514297163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[0]R generation[0..12]: [136, 143, 143, 134, 143, 143, 142, 142, 140, 142, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514317262Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514332042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[1]R flush_numbers[0..12]: [237, 245, 245, 231, 245, 245, 243, 243, 242, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514345772Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[1]R generation[0..12]: [136, 143, 143, 134, 143, 143, 142, 142, 140, 142, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514359402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514373042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[2]R flush_numbers[0..12]: [237, 245, 245, 231, 245, 245, 243, 243, 242, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514386792Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[2]R generation[0..12]: [136, 143, 143, 134, 143, 143, 142, 142, 140, 142, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514400722Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514419002Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514432301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514445132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514457911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514470501Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"":"downstairs","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514483361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"Wait for a query_work_queue command to finish before sending IO 8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514501761Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"0","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514515521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"1","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514528491Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"client":"2","":"downstairs","session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514541621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.514553691Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"bbb4303d-5e81-4677-9bff-13442bd365ac is now active with session: 8e53f39d-4690-4fa4-903e-1c33ab7f8605","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.5145664Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.51457832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768,"session_id":"8e53f39d-4690-4fa4-903e-1c33ab7f8605"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.5145906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} {"msg":"Activated sub_volume bbb4303d-5e81-4677-9bff-13442bd365ac","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:02.5146018Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 301 len 16384 data: 10 8 7 8 02/30 Read block 327 len 8192 03/30 Write block 64 len 28672 data: 8 10 13 13 13 14 12 04/30 Read block 41 len 8192 05/30 Read block 8 len 40960 06/30 Read block 341 len 28672 07/30 Read block 36 len 16384 08/30 Write block 21 len 8192 data: 16 16 09/30 Flush 10/30 Write block 455 len 8192 data: 11 11 11/30 Read block 138 len 20480 12/30 Read block 339 len 12288 13/30 Read block 112 len 28672 14/30 Write block 220 len 16384 data: 9 11 9 9 15/30 Write block 497 len 12288 data: 11 12 12 16/30 Read block 107 len 12288 17/30 Write block 79 len 40960 data: 11 8 10 10 9 11 10 10 12 10 18/30 Write block 290 len 40960 data: 15 16 17 18 13 10 7 8 7 10 19/30 Read block 311 len 16384 20/30 Flush 21/30 Write block 432 len 28672 data: 15 15 15 15 12 11 13 22/30 Read block 218 len 20480 23/30 Write block 381 len 24576 data: 10 9 10 13 16 17 24/30 Read block 321 len 36864 25/30 Flush 26/30 Write block 216 len 36864 data: 13 12 11 10 10 12 10 10 9 27/30 Read block 321 len 28672 28/30 Read block 57 len 12288 29/30 Write block 445 len 24576 data: 7 9 9 9 10 8 30/30 Write block 9 len 8192 data: 15 15 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:02.535184546Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1768} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 237 237 237 T T T 1 020-039 144 144 144 246 246 246 F F F 2 040-059 143 143 143 245 245 245 F F F 3 060-079 144 144 144 247 247 247 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 143 143 143 245 245 245 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 142 142 142 243 243 243 F F F 8 160-179 140 140 140 242 242 242 F F F 9 180-199 142 142 142 243 243 243 F F F 10 200-219 133 133 133 228 228 228 T T T 11 220-239 144 144 144 247 247 247 T T T 12 240-259 143 143 143 245 245 245 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 144 144 144 247 247 247 F F F 15 300-319 144 144 144 246 246 246 F F F 16 320-339 140 140 140 242 242 242 F F F 17 340-359 143 143 143 245 245 245 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 144 144 144 248 248 248 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 144 144 144 248 248 248 F F F 22 440-459 144 144 144 247 247 247 T T T 23 460-479 139 139 139 241 241 241 F F F 24 480-499 144 144 144 247 247 247 F F F 25 500-519 140 140 140 242 242 242 F F F 26 520-539 142 142 142 243 243 243 F F F 27 540-559 142 142 142 243 243 243 F F F 28 560-579 143 143 143 245 245 245 F F F 29 580-599 142 142 142 244 244 244 F F F Max gen: 144, Max flush: 248 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 F F F 1 020-039 144 144 144 246 246 246 F F F 2 040-059 143 143 143 245 245 245 F F F 3 060-079 144 144 144 247 247 247 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 143 143 143 245 245 245 F F F 6 120-139 142 142 142 243 243 243 F F F 7 140-159 142 142 142 243 243 243 F F F 8 160-179 140 140 140 242 242 242 F F F 9 180-199 142 142 142 243 243 243 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 143 143 143 245 245 245 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 144 144 144 247 247 247 F F F 15 300-319 144 144 144 246 246 246 F F F 16 320-339 140 140 140 242 242 242 F F F 17 340-359 143 143 143 245 245 245 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 144 144 144 248 248 248 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 144 144 144 248 248 248 F F F 22 440-459 145 145 145 249 249 249 F F F 23 460-479 139 139 139 241 241 241 F F F 24 480-499 144 144 144 247 247 247 F F F 25 500-519 140 140 140 242 242 242 F F F 26 520-539 142 142 142 243 243 243 F F F 27 540-559 142 142 142 243 243 243 F F F 28 560-579 143 143 143 245 245 245 F F F 29 580-599 142 142 142 244 244 244 F F F Max gen: 145, Max flush: 249 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.763312695Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:06.806323243Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.80668556Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.837822497Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.838489562Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.838524592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.838538962Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.838553352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} {"msg":"Crucible 92d03492-9507-4046-b7b6-10ff40134da6 has session id: b12c9454-f73a-471e-a9ab-45babe59652a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839148337Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Upstairs opts: Upstairs UUID: 92d03492-9507-4046-b7b6-10ff40134da6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839179877Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Crucible stats registered with UUID: 92d03492-9507-4046-b7b6-10ff40134da6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839195347Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:06.839209087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839663443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839688033Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} {"msg":"92d03492-9507-4046-b7b6-10ff40134da6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839738343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839766203Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"0","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839796022Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"0","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839810402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"1","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839824612Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"1","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839837502Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"2","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839851572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"2","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839870972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839945921Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.839976341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36251)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.84004254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36441)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.84008292Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40874)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.840163949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.840720445Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"2","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.840760705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"1","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.840879044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"0","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.841361851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"1","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.84144062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"2","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.841688348Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"0","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.841973036Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842045655Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[0]R flush_numbers[0..12]: [249, 246, 245, 247, 247, 245, 243, 243, 242, 243, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842241874Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[0]R generation[0..12]: [145, 144, 143, 144, 144, 143, 142, 142, 140, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842266554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842282064Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[1]R flush_numbers[0..12]: [249, 246, 245, 247, 247, 245, 243, 243, 242, 243, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842296104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[1]R generation[0..12]: [145, 144, 143, 144, 144, 143, 142, 142, 140, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842309773Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842323833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[2]R flush_numbers[0..12]: [249, 246, 245, 247, 247, 245, 243, 243, 242, 243, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842337603Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[2]R generation[0..12]: [145, 144, 143, 144, 144, 143, 142, 142, 140, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842351163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842364763Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842377513Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842391283Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:07:06.842404233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842422672Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"":"downstairs","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842435883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842447862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"0","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842461352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"1","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842476122Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"client":"2","":"downstairs","session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842489792Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842502022Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"92d03492-9507-4046-b7b6-10ff40134da6 is now active with session: b12c9454-f73a-471e-a9ab-45babe59652a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842514862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842526682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776,"session_id":"b12c9454-f73a-471e-a9ab-45babe59652a"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842538812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} {"msg":"Activated sub_volume 92d03492-9507-4046-b7b6-10ff40134da6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:06.842550062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 514 len 36864 data: 6 8 9 11 12 15 15 17 17 03/30 Write block 548 len 12288 data: 16 17 12 04/30 Write block 579 len 12288 data: 14 15 12 05/30 Read block 421 len 16384 06/30 Read block 329 len 16384 07/30 Read block 364 len 20480 08/30 Write block 571 len 4096 data: 8 09/30 Read block 322 len 24576 10/30 Read block 353 len 28672 11/30 Write block 249 len 28672 data: 14 14 11 10 10 14 12 12/30 Read block 554 len 28672 13/30 Write block 250 len 40960 data: 15 12 11 11 15 13 8 7 8 7 14/30 Write block 242 len 28672 data: 9 9 9 8 8 9 13 15/30 Write block 321 len 40960 data: 16 12 14 16 15 17 17 12 10 12 16/30 Write block 385 len 32768 data: 17 18 13 13 15 14 11 9 17/30 Write block 314 len 20480 data: 6 8 10 9 10 18/30 Write block 329 len 36864 data: 11 13 10 11 9 10 11 12 14 19/30 Read block 318 len 4096 20/30 Write block 25 len 40960 data: 13 16 13 11 7 8 12 12 13 15 21/30 Write block 152 len 28672 data: 16 14 12 10 11 9 12 22/30 Write block 412 len 36864 data: 10 11 7 7 7 7 7 9 9 23/30 Write block 336 len 4096 data: 13 24/30 Read block 232 len 28672 25/30 Read block 430 len 8192 26/30 Read block 81 len 28672 27/30 Write block 63 len 40960 data: 7 9 11 14 14 14 15 13 12 14 28/30 Write block 129 len 24576 data: 14 15 20 18 17 16 29/30 Write block 469 len 4096 data: 10 30/30 Read block 430 len 28672 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 3 4 Done 29 28 26 83 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:06.862156595Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1776} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 F F F 1 020-039 144 144 144 246 246 246 T T T 2 040-059 143 143 143 245 245 245 F F F 3 060-079 144 144 144 247 247 247 T T T 4 080-099 144 144 144 247 247 247 F F F 5 100-119 143 143 143 245 245 245 F F F 6 120-139 142 142 142 243 243 243 T T T 7 140-159 142 142 142 243 243 243 T T T 8 160-179 140 140 140 242 242 242 F F F 9 180-199 142 142 142 243 243 243 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 143 143 143 245 245 245 T T T 13 260-279 140 140 140 242 242 242 F F F 14 280-299 144 144 144 247 247 247 F F F 15 300-319 144 144 144 246 246 246 T T T 16 320-339 140 140 140 242 242 242 T T T 17 340-359 143 143 143 245 245 245 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 144 144 144 248 248 248 T T T 20 400-419 138 138 138 239 239 239 T T T 21 420-439 144 144 144 248 248 248 T T T 22 440-459 145 145 145 249 249 249 F F F 23 460-479 139 139 139 241 241 241 T T T 24 480-499 144 144 144 247 247 247 F F F 25 500-519 140 140 140 242 242 242 T T T 26 520-539 142 142 142 243 243 243 T T T 27 540-559 142 142 142 243 243 243 T T T 28 560-579 143 143 143 245 245 245 T T T 29 580-599 142 142 142 244 244 244 T T T Max gen: 145, Max flush: 249 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 F F F 1 020-039 146 146 146 250 250 250 F F F 2 040-059 143 143 143 245 245 245 F F F 3 060-079 146 146 146 250 250 250 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 143 143 143 245 245 245 F F F 6 120-139 146 146 146 250 250 250 F F F 7 140-159 146 146 146 250 250 250 F F F 8 160-179 140 140 140 242 242 242 F F F 9 180-199 142 142 142 243 243 243 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 146 146 146 250 250 250 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 144 144 144 247 247 247 F F F 15 300-319 146 146 146 250 250 250 F F F 16 320-339 146 146 146 250 250 250 F F F 17 340-359 143 143 143 245 245 245 F F F 18 360-379 138 138 138 240 240 240 F F F 19 380-399 146 146 146 250 250 250 F F F 20 400-419 146 146 146 250 250 250 F F F 21 420-439 146 146 146 250 250 250 F F F 22 440-459 145 145 145 249 249 249 F F F 23 460-479 146 146 146 250 250 250 F F F 24 480-499 144 144 144 247 247 247 F F F 25 500-519 146 146 146 250 250 250 F F F 26 520-539 146 146 146 250 250 250 F F F 27 540-559 146 146 146 250 250 250 F F F 28 560-579 146 146 146 250 250 250 F F F 29 580-599 146 146 146 250 250 250 F F F Max gen: 146, Max flush: 250 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.184150441Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.227841884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.228403089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.228456089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.228474279Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.228492509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} {"msg":"Crucible a59d4749-8b2f-4159-8927-c2d4d76c2a29 has session id: c6b1981d-1bc5-4d14-9998-d36eb8936fb4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229046835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Upstairs opts: Upstairs UUID: a59d4749-8b2f-4159-8927-c2d4d76c2a29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229094244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Crucible stats registered with UUID: a59d4749-8b2f-4159-8927-c2d4d76c2a29","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229122364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:11.229140084Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229565761Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.2295901Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} {"msg":"a59d4749-8b2f-4159-8927-c2d4d76c2a29 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.22965567Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.22968827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"0","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229714799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"0","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229729539Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229744029Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229757889Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"2","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229783819Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"2","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229809589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229829989Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229843969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49206)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229857909Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:11.229933508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:11.229982077Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:11.230078877Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64054)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230113816Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230131906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230635833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"2","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230663672Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"0","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231273128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"2","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231302028Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"0","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231889443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.232003042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.231177696Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35291)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.231392864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.232504176Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233043852Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[0]R flush_numbers[0..12]: [249, 250, 245, 250, 247, 245, 250, 250, 242, 243, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233725107Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[0]R generation[0..12]: [145, 146, 143, 146, 144, 143, 146, 146, 140, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233748177Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233764046Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[1]R flush_numbers[0..12]: [249, 250, 245, 250, 247, 245, 250, 250, 242, 243, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233779106Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[1]R generation[0..12]: [145, 146, 143, 146, 144, 143, 146, 146, 140, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233793166Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233807036Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[2]R flush_numbers[0..12]: [249, 250, 245, 250, 247, 245, 250, 250, 242, 243, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233820806Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[2]R generation[0..12]: [145, 146, 143, 146, 144, 143, 146, 146, 140, 142, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233834446Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233848356Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Max found gen is 147","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233861186Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Generation requested: 148 >= found:147","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233874396Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233887356Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233900035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"":"downstairs","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233913635Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233929205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"0","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233943085Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"1","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233956305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"client":"2","":"downstairs","session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233970115Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233982345Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"a59d4749-8b2f-4159-8927-c2d4d76c2a29 is now active with session: c6b1981d-1bc5-4d14-9998-d36eb8936fb4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.233995085Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.234007124Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784,"session_id":"c6b1981d-1bc5-4d14-9998-d36eb8936fb4"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.234019404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} {"msg":"Activated sub_volume a59d4749-8b2f-4159-8927-c2d4d76c2a29","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:21.234030644Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 517 len 20480 data: 12 13 16 16 18 02/30 Write block 546 len 40960 data: 16 20 17 18 13 8 12 13 12 16 03/30 Write block 357 len 32768 data: 13 11 13 10 8 6 7 9 04/30 Write block 582 len 36864 data: 13 12 11 12 13 14 13 12 11 05/30 Write block 361 len 8192 data: 9 7 06/30 Read block 321 len 8192 07/30 Read block 209 len 40960 08/30 Write block 510 len 32768 data: 13 13 10 11 7 9 10 13 09/30 Write block 301 len 40960 data: 11 9 8 9 8 7 7 6 6 6 10/30 Read block 320 len 28672 11/30 Read block 30 len 24576 12/30 Read block 141 len 40960 13/30 Read block 319 len 12288 14/30 Read block 437 len 20480 15/30 Write block 172 len 36864 data: 10 12 11 11 12 11 10 12 12 16/30 Write block 488 len 32768 data: 15 17 16 17 20 19 16 14 17/30 Flush 18/30 Write block 338 len 32768 data: 15 13 11 9 9 10 10 10 19/30 Read block 335 len 8192 20/30 Read block 368 len 20480 21/30 Write block 170 len 36864 data: 12 9 11 13 12 12 13 12 11 22/30 Read block 286 len 36864 23/30 Write block 414 len 40960 data: 8 8 8 8 8 10 10 12 12 13 24/30 Read block 387 len 16384 25/30 Read block 419 len 16384 26/30 Write block 162 len 4096 data: 8 27/30 Write block 500 len 24576 data: 11 11 8 10 10 10 28/30 Read block 376 len 8192 29/30 Read block 93 len 40960 30/30 Read block 146 len 40960 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:21.256762864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 F F F 1 020-039 146 146 146 250 250 250 F F F 2 040-059 143 143 143 245 245 245 F F F 3 060-079 146 146 146 250 250 250 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 143 143 143 245 245 245 F F F 6 120-139 146 146 146 250 250 250 F F F 7 140-159 146 146 146 250 250 250 F F F 8 160-179 148 148 148 251 251 251 T T T 9 180-199 148 148 148 251 251 251 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 146 146 146 250 250 250 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 144 144 144 247 247 247 F F F 15 300-319 148 148 148 251 251 251 F F F 16 320-339 146 146 146 250 250 250 T T T 17 340-359 148 148 148 251 251 251 T T T 18 360-379 148 148 148 251 251 251 F F F 19 380-399 146 146 146 250 250 250 F F F 20 400-419 146 146 146 250 250 250 T T T 21 420-439 146 146 146 250 250 250 T T T 22 440-459 145 145 145 249 249 249 F F F 23 460-479 146 146 146 250 250 250 F F F 24 480-499 148 148 148 251 251 251 F F F 25 500-519 148 148 148 251 251 251 T T T 26 520-539 148 148 148 251 251 251 F F F 27 540-559 148 148 148 251 251 251 F F F 28 560-579 146 146 146 250 250 250 F F F 29 580-599 148 148 148 251 251 251 F F F Max gen: 148, Max flush: 251 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 F F F 1 020-039 146 146 146 250 250 250 F F F 2 040-059 143 143 143 245 245 245 F F F 3 060-079 146 146 146 250 250 250 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 143 143 143 245 245 245 F F F 6 120-139 146 146 146 250 250 250 F F F 7 140-159 146 146 146 250 250 250 F F F 8 160-179 149 149 149 252 252 252 F F F 9 180-199 148 148 148 251 251 251 F F F 10 200-219 145 145 145 249 249 249 F F F 11 220-239 145 145 145 249 249 249 F F F 12 240-259 146 146 146 250 250 250 F F F 13 260-279 140 140 140 242 242 242 F F F 14 280-299 144 144 144 247 247 247 F F F 15 300-319 148 148 148 251 251 251 F F F 16 320-339 149 149 149 252 252 252 F F F 17 340-359 149 149 149 252 252 252 F F F 18 360-379 148 148 148 251 251 251 F F F 19 380-399 146 146 146 250 250 250 F F F 20 400-419 149 149 149 252 252 252 F F F 21 420-439 149 149 149 252 252 252 F F F 22 440-459 145 145 145 249 249 249 F F F 23 460-479 146 146 146 250 250 250 F F F 24 480-499 148 148 148 251 251 251 F F F 25 500-519 149 149 149 252 252 252 F F F 26 520-539 148 148 148 251 251 251 F F F 27 540-559 148 148 148 251 251 251 F F F 28 560-579 146 146 146 250 250 250 F F F 29 580-599 148 148 148 251 251 251 F F F Max gen: 149, Max flush: 252 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.499355335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:25.542272063Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.54265949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.573911916Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.574505062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.574542221Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.574558801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.574576311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} {"msg":"Crucible 1eaea1ba-d655-4777-a54a-84753a3df0f5 has session id: 3a643871-7926-400d-b914-548fd06fb88f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575290146Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Upstairs opts: Upstairs UUID: 1eaea1ba-d655-4777-a54a-84753a3df0f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575330366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Crucible stats registered with UUID: 1eaea1ba-d655-4777-a54a-84753a3df0f5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575352106Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:25.575366055Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575749772Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575773972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} {"msg":"1eaea1ba-d655-4777-a54a-84753a3df0f5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575842862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575861012Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"0","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575888271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"0","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575902291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"1","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575922271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"1","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575946781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"2","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575972661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"2","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.575988091Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.57604499Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.57607457Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39363)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.57610307Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63033)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.57615264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46962)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.576248599Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.576874564Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"2","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.577051353Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"1","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.577091922Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"0","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.57743154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"2","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.577549329Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"1","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.577836727Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"0","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578031945Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578123535Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[0]R flush_numbers[0..12]: [249, 250, 245, 250, 247, 245, 250, 250, 252, 251, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578385113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[0]R generation[0..12]: [145, 146, 143, 146, 144, 143, 146, 146, 149, 148, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578407503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578422623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[1]R flush_numbers[0..12]: [249, 250, 245, 250, 247, 245, 250, 250, 252, 251, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578436702Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[1]R generation[0..12]: [145, 146, 143, 146, 144, 143, 146, 146, 149, 148, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578454192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578468472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[2]R flush_numbers[0..12]: [249, 250, 245, 250, 247, 245, 250, 250, 252, 251, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578482322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[2]R generation[0..12]: [145, 146, 143, 146, 144, 143, 146, 146, 149, 148, 145, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578495742Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578509452Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578522172Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578535311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:07:25.578548411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578567891Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"":"downstairs","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578580911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578592801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"0","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578606001Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"1","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578618801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"client":"2","":"downstairs","session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578631731Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578643821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"1eaea1ba-d655-4777-a54a-84753a3df0f5 is now active with session: 3a643871-7926-400d-b914-548fd06fb88f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.578658771Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.57867085Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792,"session_id":"3a643871-7926-400d-b914-548fd06fb88f"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-18T05:07:25.57868321Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} {"msg":"Activated sub_volume 1eaea1ba-d655-4777-a54a-84753a3df0f5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:25.5786978Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 215 len 32768 data: 14 14 13 12 11 11 13 11 02/30 Write block 300 len 16384 data: 10 12 10 9 03/30 Write block 239 len 40960 data: 10 9 10 10 10 10 9 9 10 14 04/30 Read block 70 len 40960 05/30 Write block 322 len 16384 data: 13 15 17 16 06/30 Read block 66 len 4096 07/30 Write block 351 len 32768 data: 19 18 17 19 14 15 14 12 08/30 Write block 43 len 28672 data: 11 9 7 6 8 9 9 09/30 Write block 159 len 4096 data: 11 10/30 Read block 400 len 16384 11/30 Write block 279 len 12288 data: 9 10 11 12/30 Write block 496 len 28672 data: 10 12 13 13 12 12 9 13/30 Write block 75 len 20480 data: 16 12 11 12 12 14/30 Read block 285 len 8192 15/30 Write block 119 len 28672 data: 13 13 12 13 15 17 16 16/30 Read block 178 len 40960 17/30 Read block 325 len 28672 18/30 Write block 290 len 8192 data: 16 17 19/30 Read block 473 len 8192 20/30 Write block 489 len 12288 data: 18 17 18 21/30 Read block 515 len 4096 22/30 Flush 23/30 Write block 307 len 40960 data: 8 7 7 7 7 8 7 7 9 11 24/30 Flush 25/30 Write block 586 len 32768 data: 14 15 14 13 12 9 10 10 26/30 Read block 174 len 28672 27/30 Write block 294 len 20480 data: 14 11 8 9 8 28/30 Read block 560 len 20480 29/30 Write block 329 len 8192 data: 12 14 30/30 Read block 150 len 4096 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:25.598055536Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1792} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 F F F 1 020-039 146 146 146 250 250 250 F F F 2 040-059 150 150 150 253 253 253 F F F 3 060-079 150 150 150 253 253 253 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 150 150 150 253 253 253 F F F 6 120-139 150 150 150 253 253 253 F F F 7 140-159 150 150 150 253 253 253 F F F 8 160-179 149 149 149 252 252 252 F F F 9 180-199 148 148 148 251 251 251 F F F 10 200-219 150 150 150 253 253 253 F F F 11 220-239 150 150 150 253 253 253 F F F 12 240-259 150 150 150 253 253 253 F F F 13 260-279 150 150 150 253 253 253 F F F 14 280-299 150 150 150 253 253 253 T T T 15 300-319 150 150 150 254 254 254 F F F 16 320-339 150 150 150 253 253 253 T T T 17 340-359 150 150 150 253 253 253 F F F 18 360-379 148 148 148 251 251 251 F F F 19 380-399 146 146 146 250 250 250 F F F 20 400-419 149 149 149 252 252 252 F F F 21 420-439 149 149 149 252 252 252 F F F 22 440-459 145 145 145 249 249 249 F F F 23 460-479 146 146 146 250 250 250 F F F 24 480-499 150 150 150 253 253 253 F F F 25 500-519 150 150 150 253 253 253 F F F 26 520-539 148 148 148 251 251 251 F F F 27 540-559 148 148 148 251 251 251 F F F 28 560-579 146 146 146 250 250 250 F F F 29 580-599 148 148 148 251 251 251 T T T Max gen: 150, Max flush: 254 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 F F F 1 020-039 146 146 146 250 250 250 F F F 2 040-059 150 150 150 253 253 253 F F F 3 060-079 150 150 150 253 253 253 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 150 150 150 253 253 253 F F F 6 120-139 150 150 150 253 253 253 F F F 7 140-159 150 150 150 253 253 253 F F F 8 160-179 149 149 149 252 252 252 F F F 9 180-199 148 148 148 251 251 251 F F F 10 200-219 150 150 150 253 253 253 F F F 11 220-239 150 150 150 253 253 253 F F F 12 240-259 150 150 150 253 253 253 F F F 13 260-279 150 150 150 253 253 253 F F F 14 280-299 151 151 151 255 255 255 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 F F F 17 340-359 150 150 150 253 253 253 F F F 18 360-379 148 148 148 251 251 251 F F F 19 380-399 146 146 146 250 250 250 F F F 20 400-419 149 149 149 252 252 252 F F F 21 420-439 149 149 149 252 252 252 F F F 22 440-459 145 145 145 249 249 249 F F F 23 460-479 146 146 146 250 250 250 F F F 24 480-499 150 150 150 253 253 253 F F F 25 500-519 150 150 150 253 253 253 F F F 26 520-539 148 148 148 251 251 251 F F F 27 540-559 148 148 148 251 251 251 F F F 28 560-579 146 146 146 250 250 250 F F F 29 580-599 151 151 151 255 255 255 F F F Max gen: 151, Max flush: 255 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.858352334Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.902377544Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.903067529Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.903112208Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.903135368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.903153808Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} {"msg":"Crucible 9df9c4af-8215-42f9-a35e-66ea0ac9a212 has session id: e6f7150c-a58b-4bfa-9e2b-afa3d94033b1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.903751974Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Upstairs opts: Upstairs UUID: 9df9c4af-8215-42f9-a35e-66ea0ac9a212, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.903794503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Crucible stats registered with UUID: 9df9c4af-8215-42f9-a35e-66ea0ac9a212","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.903818833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:29.903839593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904307669Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904340039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} {"msg":"9df9c4af-8215-42f9-a35e-66ea0ac9a212 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904398439Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904428709Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"0","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904457558Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"0","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904471928Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"1","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904486118Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"1","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904503228Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"2","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904517678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"2","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904533448Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904557677Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904609427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36320)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904635617Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47873)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904659997Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45241)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.904762116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.905294322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"1","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.905456161Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"0","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.905817498Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"1","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.905992817Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"0","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.906436253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.906544933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.906663132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"2","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907179508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"2","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[0]R flush_numbers[0..12]: [249, 250, 253, 253, 247, 253, 253, 253, 252, 251, 253, 253]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907691874Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[0]R generation[0..12]: [145, 146, 150, 150, 144, 150, 150, 150, 149, 148, 150, 150]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907717154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907733974Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[1]R flush_numbers[0..12]: [249, 250, 253, 253, 247, 253, 253, 253, 252, 251, 253, 253]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907748473Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[1]R generation[0..12]: [145, 146, 150, 150, 144, 150, 150, 150, 149, 148, 150, 150]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907762244Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907776113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[2]R flush_numbers[0..12]: [249, 250, 253, 253, 247, 253, 253, 253, 252, 251, 253, 253]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907789883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[2]R generation[0..12]: [145, 146, 150, 150, 144, 150, 150, 150, 149, 148, 150, 150]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907803353Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907817343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907830313Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907843213Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-18T05:07:29.907861803Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907879363Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"":"downstairs","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907893423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907906622Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"0","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907919922Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"1","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907932882Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"client":"2","":"downstairs","session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907945712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907957772Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"9df9c4af-8215-42f9-a35e-66ea0ac9a212 is now active with session: e6f7150c-a58b-4bfa-9e2b-afa3d94033b1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907970582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.907982512Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800,"session_id":"e6f7150c-a58b-4bfa-9e2b-afa3d94033b1"} {"msg":"The guest has finished waiting for activation with:152","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2025-03-18T05:07:29.908009492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} {"msg":"Activated sub_volume 9df9c4af-8215-42f9-a35e-66ea0ac9a212","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:29.908024062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 269 len 4096 data: 11 03/30 Write block 11 len 4096 data: 17 04/30 Read block 482 len 20480 05/30 Write block 226 len 8192 data: 9 10 06/30 Write block 389 len 4096 data: 16 07/30 Read block 382 len 28672 08/30 Read block 221 len 32768 09/30 Write block 252 len 4096 data: 12 10/30 Read block 239 len 16384 11/30 Write block 107 len 16384 data: 6 8 11 9 12/30 Write block 223 len 32768 data: 11 10 10 10 11 12 10 11 13/30 Write block 16 len 12288 data: 15 16 14 14/30 Read block 266 len 32768 15/30 Read block 28 len 8192 16/30 Read block 269 len 40960 17/30 Read block 557 len 12288 18/30 Read block 7 len 16384 19/30 Write block 522 len 4096 data: 18 20/30 Write block 49 len 24576 data: 10 7 8 8 9 9 21/30 Read block 162 len 24576 22/30 Read block 491 len 8192 23/30 Write block 73 len 24576 data: 15 13 17 13 12 13 24/30 Write block 464 len 36864 data: 11 11 9 10 10 11 12 10 8 25/30 Write block 436 len 32768 data: 13 12 14 12 9 5 6 8 26/30 Write block 533 len 40960 data: 8 8 6 5 6 8 10 11 13 12 27/30 Read block 124 len 20480 28/30 Read block 466 len 20480 29/30 Write block 158 len 8192 data: 13 12 30/30 Read block 455 len 28672 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 28 28 29 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:29.926144506Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1800} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 249 249 249 T T T 1 020-039 146 146 146 250 250 250 F F F 2 040-059 150 150 150 253 253 253 T T T 3 060-079 150 150 150 253 253 253 T T T 4 080-099 144 144 144 247 247 247 F F F 5 100-119 150 150 150 253 253 253 T T T 6 120-139 150 150 150 253 253 253 F F F 7 140-159 150 150 150 253 253 253 T T T 8 160-179 149 149 149 252 252 252 F F F 9 180-199 148 148 148 251 251 251 F F F 10 200-219 150 150 150 253 253 253 F F F 11 220-239 150 150 150 253 253 253 T T T 12 240-259 150 150 150 253 253 253 T T T 13 260-279 150 150 150 253 253 253 T T T 14 280-299 151 151 151 255 255 255 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 F F F 17 340-359 150 150 150 253 253 253 F F F 18 360-379 148 148 148 251 251 251 F F F 19 380-399 146 146 146 250 250 250 T T T 20 400-419 149 149 149 252 252 252 F F F 21 420-439 149 149 149 252 252 252 T T T 22 440-459 145 145 145 249 249 249 T T T 23 460-479 146 146 146 250 250 250 T T T 24 480-499 150 150 150 253 253 253 F F F 25 500-519 150 150 150 253 253 253 F F F 26 520-539 148 148 148 251 251 251 T T T 27 540-559 148 148 148 251 251 251 T T T 28 560-579 146 146 146 250 250 250 F F F 29 580-599 151 151 151 255 255 255 F F F Max gen: 151, Max flush: 255 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 256 256 256 F F F 1 020-039 146 146 146 250 250 250 F F F 2 040-059 152 152 152 256 256 256 F F F 3 060-079 152 152 152 256 256 256 F F F 4 080-099 144 144 144 247 247 247 F F F 5 100-119 152 152 152 256 256 256 F F F 6 120-139 150 150 150 253 253 253 F F F 7 140-159 152 152 152 256 256 256 F F F 8 160-179 149 149 149 252 252 252 F F F 9 180-199 148 148 148 251 251 251 F F F 10 200-219 150 150 150 253 253 253 F F F 11 220-239 152 152 152 256 256 256 F F F 12 240-259 152 152 152 256 256 256 F F F 13 260-279 152 152 152 256 256 256 F F F 14 280-299 151 151 151 255 255 255 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 F F F 17 340-359 150 150 150 253 253 253 F F F 18 360-379 148 148 148 251 251 251 F F F 19 380-399 152 152 152 256 256 256 F F F 20 400-419 149 149 149 252 252 252 F F F 21 420-439 152 152 152 256 256 256 F F F 22 440-459 152 152 152 256 256 256 F F F 23 460-479 152 152 152 256 256 256 F F F 24 480-499 150 150 150 253 253 253 F F F 25 500-519 150 150 150 253 253 253 F F F 26 520-539 152 152 152 256 256 256 F F F 27 540-559 152 152 152 256 256 256 F F F 28 560-579 146 146 146 250 250 250 F F F 29 580-599 151 151 151 255 255 255 F F F Max gen: 152, Max flush: 256 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.207945711Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.251877482Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.252427538Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.252469448Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.252484098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.252498158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} {"msg":"Crucible 5b9daf63-ccbe-4e4a-a526-f63ef7381a2a has session id: 5f63eebf-34ef-4ccb-a740-8d5d75a10100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.252969364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Upstairs opts: Upstairs UUID: 5b9daf63-ccbe-4e4a-a526-f63ef7381a2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253012724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Crucible stats registered with UUID: 5b9daf63-ccbe-4e4a-a526-f63ef7381a2a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253036874Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:44.253058094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25351125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25355047Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} {"msg":"5b9daf63-ccbe-4e4a-a526-f63ef7381a2a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25358687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25360061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"0","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25362592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"0","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253639849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"1","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253654169Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"1","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253667319Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"2","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253681229Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"2","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253694299Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253716539Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253731589Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"ds_connection connected from Ok(127.0.0.1:65194)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253790658Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"ds_connection connected from Ok(127.0.0.1:56816)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.253907387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"ds_connection connected from Ok(127.0.0.1:48327)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.254071456Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.254370864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"0","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.254488413Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"1","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25485156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"0","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.254978349Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"1","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.255443616Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.255567625Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.256146311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"2","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.256689797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"2","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[0]R flush_numbers[0..12]: [256, 250, 256, 256, 247, 256, 253, 256, 252, 251, 253, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257231002Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[0]R generation[0..12]: [152, 146, 152, 152, 144, 152, 150, 152, 149, 148, 150, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257264802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257284372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[1]R flush_numbers[0..12]: [256, 250, 256, 256, 247, 256, 253, 256, 252, 251, 253, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257305752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[1]R generation[0..12]: [152, 146, 152, 152, 144, 152, 150, 152, 149, 148, 150, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257323312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257340762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[2]R flush_numbers[0..12]: [256, 250, 256, 256, 247, 256, 253, 256, 252, 251, 253, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257357821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[2]R generation[0..12]: [152, 146, 152, 152, 144, 152, 150, 152, 149, 148, 150, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257374681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257391601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Max found gen is 153","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257407471Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Generation requested: 154 >= found:153","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257423971Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257440241Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257456371Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"":"downstairs","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.257472591Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25748764Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"0","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25751267Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"1","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25752592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"client":"2","":"downstairs","session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25753902Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.2575534Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"5b9daf63-ccbe-4e4a-a526-f63ef7381a2a is now active with session: 5f63eebf-34ef-4ccb-a740-8d5d75a10100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25756647Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25757857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808,"session_id":"5f63eebf-34ef-4ccb-a740-8d5d75a10100"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25759094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} {"msg":"Activated sub_volume 5b9daf63-ccbe-4e4a-a526-f63ef7381a2a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:44.25760242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 91 len 16384 data: 12 15 16 17 02/30 Read block 456 len 32768 03/30 Flush 04/30 Write block 492 len 40960 data: 21 20 17 15 11 13 14 14 13 13 05/30 Write block 419 len 12288 data: 11 11 13 06/30 Read block 234 len 4096 07/30 Write block 481 len 28672 data: 8 14 16 16 16 18 17 08/30 Write block 114 len 4096 data: 11 09/30 Write block 94 len 28672 data: 18 14 12 10 8 10 11 10/30 Write block 261 len 32768 data: 9 9 12 9 13 11 11 11 11/30 Read block 485 len 24576 12/30 Write block 343 len 28672 data: 11 11 11 11 14 10 13 13/30 Write block 163 len 36864 data: 9 7 9 13 13 9 12 13 10 14/30 Write block 187 len 16384 data: 13 9 10 11 15/30 Write block 124 len 4096 data: 18 16/30 Read block 131 len 32768 17/30 Read block 456 len 16384 18/30 Read block 91 len 20480 19/30 Write block 468 len 40960 data: 11 12 13 11 9 7 8 7 6 5 20/30 Write block 409 len 8192 data: 12 11 21/30 Write block 279 len 40960 data: 10 11 12 11 10 12 14 13 15 15 22/30 Read block 58 len 36864 23/30 Read block 83 len 24576 24/30 Read block 427 len 40960 25/30 Write block 594 len 20480 data: 9 11 10 10 7 26/30 Read block 483 len 16384 27/30 Read block 324 len 4096 28/30 Write block 569 len 32768 data: 9 10 9 10 9 9 11 14 29/30 Write block 31 len 20480 data: 13 13 14 16 17 30/30 Write block 528 len 20480 data: 9 10 8 8 7 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:44.276969725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1808} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 256 256 256 F F F 1 020-039 146 146 146 250 250 250 T T T 2 040-059 152 152 152 256 256 256 F F F 3 060-079 152 152 152 256 256 256 F F F 4 080-099 154 154 154 257 257 257 T T T 5 100-119 152 152 152 256 256 256 T T T 6 120-139 150 150 150 253 253 253 T T T 7 140-159 152 152 152 256 256 256 F F F 8 160-179 149 149 149 252 252 252 T T T 9 180-199 148 148 148 251 251 251 T T T 10 200-219 150 150 150 253 253 253 F F F 11 220-239 152 152 152 256 256 256 F F F 12 240-259 152 152 152 256 256 256 F F F 13 260-279 152 152 152 256 256 256 T T T 14 280-299 151 151 151 255 255 255 T T T 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 F F F 17 340-359 150 150 150 253 253 253 T T T 18 360-379 148 148 148 251 251 251 F F F 19 380-399 152 152 152 256 256 256 F F F 20 400-419 149 149 149 252 252 252 T T T 21 420-439 152 152 152 256 256 256 T T T 22 440-459 152 152 152 256 256 256 F F F 23 460-479 152 152 152 256 256 256 T T T 24 480-499 150 150 150 253 253 253 T T T 25 500-519 150 150 150 253 253 253 T T T 26 520-539 152 152 152 256 256 256 T T T 27 540-559 152 152 152 256 256 256 F F F 28 560-579 146 146 146 250 250 250 T T T 29 580-599 151 151 151 255 255 255 T T T Max gen: 154, Max flush: 257 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 256 256 256 F F F 1 020-039 155 155 155 258 258 258 F F F 2 040-059 152 152 152 256 256 256 F F F 3 060-079 152 152 152 256 256 256 F F F 4 080-099 155 155 155 258 258 258 F F F 5 100-119 155 155 155 258 258 258 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 152 152 152 256 256 256 F F F 8 160-179 155 155 155 258 258 258 F F F 9 180-199 155 155 155 258 258 258 F F F 10 200-219 150 150 150 253 253 253 F F F 11 220-239 152 152 152 256 256 256 F F F 12 240-259 152 152 152 256 256 256 F F F 13 260-279 155 155 155 258 258 258 F F F 14 280-299 155 155 155 258 258 258 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 F F F 17 340-359 155 155 155 258 258 258 F F F 18 360-379 148 148 148 251 251 251 F F F 19 380-399 152 152 152 256 256 256 F F F 20 400-419 155 155 155 258 258 258 F F F 21 420-439 155 155 155 258 258 258 F F F 22 440-459 152 152 152 256 256 256 F F F 23 460-479 155 155 155 258 258 258 F F F 24 480-499 155 155 155 258 258 258 F F F 25 500-519 155 155 155 258 258 258 F F F 26 520-539 155 155 155 258 258 258 F F F 27 540-559 152 152 152 256 256 256 F F F 28 560-579 155 155 155 258 258 258 F F F 29 580-599 155 155 155 258 258 258 F F F Max gen: 155, Max flush: 258 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.576805794Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:58.619285016Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.619669784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.650559342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.651100838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.651146308Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.651164248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.651182048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} {"msg":"Crucible 26af395f-4cf4-4f51-b0d3-d41dc25bf35d has session id: d33da647-7535-4988-b28e-b5eefa55e09b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.651762323Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Upstairs opts: Upstairs UUID: 26af395f-4cf4-4f51-b0d3-d41dc25bf35d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.651802863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Crucible stats registered with UUID: 26af395f-4cf4-4f51-b0d3-d41dc25bf35d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.651819033Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:58.651839113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652308379Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652332779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} {"msg":"26af395f-4cf4-4f51-b0d3-d41dc25bf35d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652402928Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652420918Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"0","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652455628Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"0","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652482938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"1","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652552737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"1","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652590987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"2","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652619797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"2","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652645507Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652672206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652699476Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35142)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652724856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44104)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652754046Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57687)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.652786716Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.653444361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"1","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.653472851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"2","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65348837Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"0","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.654221145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"1","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.654248965Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"2","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.654264154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"0","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65481994Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65484509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[0]R flush_numbers[0..12]: [256, 258, 256, 256, 258, 258, 258, 256, 258, 258, 253, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65488814Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[0]R generation[0..12]: [152, 155, 152, 152, 155, 155, 155, 152, 155, 155, 150, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65490605Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65492019Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[1]R flush_numbers[0..12]: [256, 258, 256, 256, 258, 258, 258, 256, 258, 258, 253, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65493402Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[1]R generation[0..12]: [152, 155, 152, 152, 155, 155, 155, 152, 155, 155, 150, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.65494766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.654964859Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[2]R flush_numbers[0..12]: [256, 258, 256, 256, 258, 258, 258, 256, 258, 258, 253, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.654979049Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[2]R generation[0..12]: [152, 155, 152, 152, 155, 155, 155, 152, 155, 155, 150, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.654992649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655006959Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655019779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655032829Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655045579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655058239Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"":"downstairs","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655071368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Transition from WaitQuorum to ActiveWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655089458Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"0","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655102998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"1","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655116108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"client":"2","":"downstairs","session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655129368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655141578Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"26af395f-4cf4-4f51-b0d3-d41dc25bf35d is now active with session: d33da647-7535-4988-b28e-b5eefa55e09b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655154168Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655167728Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816,"session_id":"d33da647-7535-4988-b28e-b5eefa55e09b"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655180008Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} {"msg":"Activated sub_volume 26af395f-4cf4-4f51-b0d3-d41dc25bf35d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.655191298Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 166 len 36864 data: 14 14 10 13 14 11 12 14 13 03/30 Read block 219 len 32768 04/30 Read block 210 len 12288 05/30 Write block 146 len 20480 data: 9 9 12 13 14 06/30 Read block 224 len 8192 07/30 Write block 413 len 8192 data: 12 9 08/30 Write block 441 len 16384 data: 6 7 9 8 09/30 Read block 350 len 20480 10/30 Read block 457 len 24576 11/30 Write block 570 len 40960 data: 11 10 11 10 10 12 15 14 14 15 12/30 Read block 78 len 20480 13/30 Read block 410 len 20480 14/30 Write block 212 len 28672 data: 15 13 15 15 15 14 13 15/30 Read block 307 len 4096 16/30 Write block 294 len 12288 data: 15 12 9 17/30 Write block 291 len 32768 data: 18 18 19 16 13 10 10 9 18/30 Read block 336 len 28672 19/30 Flush 20/30 Flush 21/30 Write block 381 len 40960 data: 11 10 11 14 18 19 14 14 17 15 22/30 Read block 266 len 32768 23/30 Write block 369 len 16384 data: 8 9 8 8 24/30 Read block 578 len 12288 25/30 Read block 303 len 12288 26/30 Read block 165 len 36864 27/30 Read block 137 len 40960 28/30 Write block 36 len 24576 data: 19 17 19 19 14 14 29/30 Write block 36 len 28672 data: 20 18 20 20 15 15 11 30/30 Write block 565 len 16384 data: 10 8 7 9 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:58.67627511Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1816} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 256 256 256 F F F 1 020-039 155 155 155 258 258 258 T T T 2 040-059 152 152 152 256 256 256 T T T 3 060-079 152 152 152 256 256 256 F F F 4 080-099 155 155 155 258 258 258 F F F 5 100-119 155 155 155 258 258 258 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 156 156 156 260 260 260 F F F 8 160-179 156 156 156 260 260 260 F F F 9 180-199 155 155 155 258 258 258 F F F 10 200-219 156 156 156 260 260 260 F F F 11 220-239 152 152 152 256 256 256 F F F 12 240-259 152 152 152 256 256 256 F F F 13 260-279 155 155 155 258 258 258 F F F 14 280-299 156 156 156 260 260 260 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 F F F 17 340-359 155 155 155 258 258 258 F F F 18 360-379 148 148 148 251 251 251 T T T 19 380-399 152 152 152 256 256 256 T T T 20 400-419 156 156 156 260 260 260 F F F 21 420-439 155 155 155 258 258 258 F F F 22 440-459 156 156 156 260 260 260 F F F 23 460-479 155 155 155 258 258 258 F F F 24 480-499 155 155 155 258 258 258 F F F 25 500-519 155 155 155 258 258 258 F F F 26 520-539 155 155 155 258 258 258 F F F 27 540-559 152 152 152 256 256 256 F F F 28 560-579 156 156 156 260 260 260 T T T 29 580-599 155 155 155 258 258 258 F F F Max gen: 156, Max flush: 260 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 256 256 256 F F F 1 020-039 157 157 157 261 261 261 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 152 152 152 256 256 256 F F F 4 080-099 155 155 155 258 258 258 F F F 5 100-119 155 155 155 258 258 258 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 156 156 156 260 260 260 F F F 8 160-179 156 156 156 260 260 260 F F F 9 180-199 155 155 155 258 258 258 F F F 10 200-219 156 156 156 260 260 260 F F F 11 220-239 152 152 152 256 256 256 F F F 12 240-259 152 152 152 256 256 256 F F F 13 260-279 155 155 155 258 258 258 F F F 14 280-299 156 156 156 260 260 260 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 F F F 17 340-359 155 155 155 258 258 258 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 157 157 157 261 261 261 F F F 20 400-419 156 156 156 260 260 260 F F F 21 420-439 155 155 155 258 258 258 F F F 22 440-459 156 156 156 260 260 260 F F F 23 460-479 155 155 155 258 258 258 F F F 24 480-499 155 155 155 258 258 258 F F F 25 500-519 155 155 155 258 258 258 F F F 26 520-539 155 155 155 258 258 258 F F F 27 540-559 152 152 152 256 256 256 F F F 28 560-579 157 157 157 261 261 261 F F F 29 580-599 155 155 155 258 258 258 F F F Max gen: 157, Max flush: 261 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.945109983Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.989388381Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.989968597Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.990022577Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.990046296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.990062186Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} {"msg":"Crucible 95eeeda0-4943-4ed0-91fe-87a9c7bc9536 has session id: ff27d36c-ef46-49a4-bfe6-5783821d91be","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.990522743Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Upstairs opts: Upstairs UUID: 95eeeda0-4943-4ed0-91fe-87a9c7bc9536, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.990586812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Crucible stats registered with UUID: 95eeeda0-4943-4ed0-91fe-87a9c7bc9536","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.990605912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:02.990618832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991243447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991269767Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} {"msg":"95eeeda0-4943-4ed0-91fe-87a9c7bc9536 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991330167Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991348836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"0","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991377566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"0","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991391906Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"1","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991416746Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"1","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991446196Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"2","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991469306Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"2","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991483166Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991496826Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991510255Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"ds_connection connected from Ok(127.0.0.1:42258)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991523805Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"ds_connection connected from Ok(127.0.0.1:39344)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991734824Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"ds_connection connected from Ok(127.0.0.1:47873)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.991786943Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.99229636Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"2","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.992353609Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"0","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.992504188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"1","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.992983144Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"0","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993012874Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"2","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993076324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"1","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.9935954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.99362489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[0]R flush_numbers[0..12]: [256, 261, 261, 256, 258, 258, 258, 260, 260, 258, 260, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993692479Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[0]R generation[0..12]: [152, 157, 157, 152, 155, 155, 155, 156, 156, 155, 156, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993713509Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993728399Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[1]R flush_numbers[0..12]: [256, 261, 261, 256, 258, 258, 258, 260, 260, 258, 260, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993742249Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[1]R generation[0..12]: [152, 157, 157, 152, 155, 155, 155, 156, 156, 155, 156, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993756389Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993770338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[2]R flush_numbers[0..12]: [256, 261, 261, 256, 258, 258, 258, 260, 260, 258, 260, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993784178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[2]R generation[0..12]: [152, 157, 157, 152, 155, 155, 155, 156, 156, 155, 156, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993797658Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993811338Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993824018Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993836948Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993849858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993863328Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"":"downstairs","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993879878Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:02.993892048Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"0","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993907908Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"1","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993921157Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"client":"2","":"downstairs","session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993934447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993946637Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"95eeeda0-4943-4ed0-91fe-87a9c7bc9536 is now active with session: ff27d36c-ef46-49a4-bfe6-5783821d91be","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993959257Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993970997Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824,"session_id":"ff27d36c-ef46-49a4-bfe6-5783821d91be"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993983027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} {"msg":"Activated sub_volume 95eeeda0-4943-4ed0-91fe-87a9c7bc9536","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:02.993994177Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 275 len 40960 data: 13 8 6 9 11 12 13 12 11 13 02/30 Write block 238 len 28672 data: 10 11 10 11 11 11 11 03/30 Read block 228 len 16384 04/30 Read block 391 len 24576 05/30 Write block 476 len 40960 data: 7 6 7 10 8 9 15 17 17 17 06/30 Read block 111 len 8192 07/30 Flush 08/30 Write block 166 len 36864 data: 15 15 11 14 15 12 13 15 14 09/30 Read block 328 len 16384 10/30 Flush 11/30 Write block 144 len 24576 data: 9 10 10 10 13 14 12/30 Read block 12 len 16384 13/30 Write block 110 len 4096 data: 10 14/30 Read block 125 len 4096 15/30 Read block 35 len 8192 16/30 Write block 324 len 12288 data: 18 17 18 17/30 Write block 143 len 28672 data: 12 10 11 11 11 14 15 18/30 Write block 8 len 36864 data: 13 16 16 18 15 14 17 17 16 19/30 Read block 504 len 28672 20/30 Read block 562 len 40960 21/30 Write block 274 len 12288 data: 10 14 9 22/30 Write block 181 len 20480 data: 13 15 14 16 14 23/30 Write block 101 len 8192 data: 8 6 24/30 Write block 535 len 8192 data: 7 6 25/30 Write block 442 len 12288 data: 8 10 9 26/30 Write block 452 len 8192 data: 6 7 27/30 Write block 449 len 12288 data: 11 9 7 28/30 Write block 105 len 20480 data: 8 8 7 9 12 29/30 Write block 29 len 12288 data: 8 9 14 30/30 Read block 254 len 28672 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 6 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 19 20 19 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:03.012697207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1824} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 256 256 256 T T T 1 020-039 157 157 157 261 261 261 T T T 2 040-059 157 157 157 261 261 261 F F F 3 060-079 152 152 152 256 256 256 F F F 4 080-099 155 155 155 258 258 258 F F F 5 100-119 155 155 155 258 258 258 T T T 6 120-139 155 155 155 258 258 258 F F F 7 140-159 156 156 156 260 260 260 T T T 8 160-179 158 158 158 263 263 263 F F F 9 180-199 155 155 155 258 258 258 T T T 10 200-219 156 156 156 260 260 260 F F F 11 220-239 158 158 158 262 262 262 F F F 12 240-259 158 158 158 262 262 262 F F F 13 260-279 158 158 158 262 262 262 T T T 14 280-299 158 158 158 262 262 262 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 151 151 151 255 255 255 T T T 17 340-359 155 155 155 258 258 258 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 157 157 157 261 261 261 F F F 20 400-419 156 156 156 260 260 260 F F F 21 420-439 155 155 155 258 258 258 F F F 22 440-459 156 156 156 260 260 260 T T T 23 460-479 158 158 158 262 262 262 F F F 24 480-499 158 158 158 262 262 262 F F F 25 500-519 155 155 155 258 258 258 F F F 26 520-539 155 155 155 258 258 258 T T T 27 540-559 152 152 152 256 256 256 F F F 28 560-579 157 157 157 261 261 261 F F F 29 580-599 155 155 155 258 258 258 F F F Max gen: 158, Max flush: 263 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 264 264 264 F F F 1 020-039 159 159 159 264 264 264 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 152 152 152 256 256 256 F F F 4 080-099 155 155 155 258 258 258 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 156 156 156 260 260 260 F F F 11 220-239 158 158 158 262 262 262 F F F 12 240-259 158 158 158 262 262 262 F F F 13 260-279 159 159 159 264 264 264 F F F 14 280-299 158 158 158 262 262 262 F F F 15 300-319 150 150 150 254 254 254 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 155 155 155 258 258 258 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 157 157 157 261 261 261 F F F 20 400-419 156 156 156 260 260 260 F F F 21 420-439 155 155 155 258 258 258 F F F 22 440-459 159 159 159 264 264 264 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 158 158 158 262 262 262 F F F 25 500-519 155 155 155 258 258 258 F F F 26 520-539 159 159 159 264 264 264 F F F 27 540-559 152 152 152 256 256 256 F F F 28 560-579 157 157 157 261 261 261 F F F 29 580-599 155 155 155 258 258 258 F F F Max gen: 159, Max flush: 264 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.266631101Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:07.311502695Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.311853743Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.343149648Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.343757214Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.343792903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.343807643Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.343822713Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} {"msg":"Crucible 99309a46-e1e5-4dbb-84a5-abdd615d0fc1 has session id: f643855b-e111-4c60-8956-2874edf43bae","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344336959Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Upstairs opts: Upstairs UUID: 99309a46-e1e5-4dbb-84a5-abdd615d0fc1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344379489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Crucible stats registered with UUID: 99309a46-e1e5-4dbb-84a5-abdd615d0fc1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344401819Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:07.344419699Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344828686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344860755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} {"msg":"99309a46-e1e5-4dbb-84a5-abdd615d0fc1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344919345Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344946485Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"0","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344974175Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"0","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.344988525Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"1","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345002705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"1","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345015704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"2","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345029784Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"2","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345042754Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345060524Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345085494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"ds_connection connected from Ok(127.0.0.1:42979)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345210713Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"ds_connection connected from Ok(127.0.0.1:39545)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345259983Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"ds_connection connected from Ok(127.0.0.1:34926)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345293192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345930778Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"1","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.345972807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"2","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.346087296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"0","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.346596353Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"1","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.346724712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"2","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.346827281Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"0","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347177828Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347248358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[0]R flush_numbers[0..12]: [264, 264, 261, 256, 258, 264, 258, 264, 263, 264, 260, 262]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347358187Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[0]R generation[0..12]: [159, 159, 157, 152, 155, 159, 155, 159, 158, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347378847Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347393796Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[1]R flush_numbers[0..12]: [264, 264, 261, 256, 258, 264, 258, 264, 263, 264, 260, 262]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347412196Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[1]R generation[0..12]: [159, 159, 157, 152, 155, 159, 155, 159, 158, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347426356Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347440216Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[2]R flush_numbers[0..12]: [264, 264, 261, 256, 258, 264, 258, 264, 263, 264, 260, 262]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347453516Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[2]R generation[0..12]: [159, 159, 157, 152, 155, 159, 155, 159, 158, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347467066Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347480736Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347493436Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347506386Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347520026Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:07.347532966Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"":"downstairs","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347549575Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347561865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"0","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347575005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"1","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347587885Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"client":"2","":"downstairs","session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347600785Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347616015Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"99309a46-e1e5-4dbb-84a5-abdd615d0fc1 is now active with session: f643855b-e111-4c60-8956-2874edf43bae","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347629025Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347640795Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832,"session_id":"f643855b-e111-4c60-8956-2874edf43bae"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347652835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} {"msg":"Activated sub_volume 99309a46-e1e5-4dbb-84a5-abdd615d0fc1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:07.347663974Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 295 len 8192 data: 14 11 02/30 Write block 395 len 20480 data: 5 6 6 7 7 03/30 Read block 557 len 36864 04/30 Write block 579 len 28672 data: 16 16 13 14 13 12 13 05/30 Flush 06/30 Write block 518 len 28672 data: 14 17 17 19 19 12 12 07/30 Read block 42 len 16384 08/30 Read block 472 len 20480 09/30 Write block 392 len 36864 data: 10 8 6 6 7 7 8 8 10 10/30 Write block 557 len 40960 data: 19 16 14 15 14 10 9 9 11 9 11/30 Write block 225 len 36864 data: 11 11 12 13 11 12 9 8 7 12/30 Read block 341 len 4096 13/30 Write block 292 len 36864 data: 19 20 17 15 12 11 10 11 11 14/30 Read block 257 len 40960 15/30 Write block 300 len 4096 data: 12 16/30 Read block 370 len 32768 17/30 Read block 423 len 12288 18/30 Read block 321 len 32768 19/30 Write block 562 len 16384 data: 11 10 10 12 20/30 Write block 353 len 24576 data: 18 20 15 16 15 13 21/30 Read block 419 len 4096 22/30 Read block 221 len 40960 23/30 Read block 266 len 36864 24/30 Write block 286 len 28672 data: 14 16 16 15 17 19 20 25/30 Write block 400 len 8192 data: 11 12 26/30 Write block 441 len 24576 data: 7 9 11 10 8 10 27/30 Read block 482 len 20480 28/30 Write block 75 len 24576 data: 18 14 13 14 13 9 29/30 Read block 335 len 28672 30/30 Read block 427 len 36864 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 7 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 24 24 25 73 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:07.368385299Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1832} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 264 264 264 F F F 1 020-039 159 159 159 264 264 264 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 152 152 152 256 256 256 T T T 4 080-099 155 155 155 258 258 258 T T T 5 100-119 159 159 159 264 264 264 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 156 156 156 260 260 260 F F F 11 220-239 158 158 158 262 262 262 T T T 12 240-259 158 158 158 262 262 262 F F F 13 260-279 159 159 159 264 264 264 F F F 14 280-299 160 160 160 265 265 265 T T T 15 300-319 150 150 150 254 254 254 T T T 16 320-339 159 159 159 264 264 264 F F F 17 340-359 155 155 155 258 258 258 T T T 18 360-379 157 157 157 261 261 261 F F F 19 380-399 160 160 160 265 265 265 T T T 20 400-419 156 156 156 260 260 260 T T T 21 420-439 155 155 155 258 258 258 F F F 22 440-459 159 159 159 264 264 264 T T T 23 460-479 158 158 158 262 262 262 F F F 24 480-499 158 158 158 262 262 262 F F F 25 500-519 155 155 155 258 258 258 T T T 26 520-539 159 159 159 264 264 264 T T T 27 540-559 152 152 152 256 256 256 T T T 28 560-579 160 160 160 265 265 265 T T T 29 580-599 160 160 160 265 265 265 F F F Max gen: 160, Max flush: 265 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 264 264 264 F F F 1 020-039 159 159 159 264 264 264 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 161 161 161 266 266 266 F F F 4 080-099 161 161 161 266 266 266 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 156 156 156 260 260 260 F F F 11 220-239 161 161 161 266 266 266 F F F 12 240-259 158 158 158 262 262 262 F F F 13 260-279 159 159 159 264 264 264 F F F 14 280-299 161 161 161 266 266 266 F F F 15 300-319 161 161 161 266 266 266 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 161 161 161 266 266 266 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 161 161 161 266 266 266 F F F 20 400-419 161 161 161 266 266 266 F F F 21 420-439 155 155 155 258 258 258 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 158 158 158 262 262 262 F F F 24 480-499 158 158 158 262 262 262 F F F 25 500-519 161 161 161 266 266 266 F F F 26 520-539 161 161 161 266 266 266 F F F 27 540-559 161 161 161 266 266 266 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 160 160 160 265 265 265 F F F Max gen: 161, Max flush: 266 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.680304821Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.724244782Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.724866067Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.724919717Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.724938336Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.724957446Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} {"msg":"Crucible 1f9d774c-3f75-40d3-b197-8c181095b426 has session id: 87ac302a-d1b3-4a5a-ba83-d98adaa4c67d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.725487152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Upstairs opts: Upstairs UUID: 1f9d774c-3f75-40d3-b197-8c181095b426, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.725543972Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Crucible stats registered with UUID: 1f9d774c-3f75-40d3-b197-8c181095b426","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.725569182Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:11.725590802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726024358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726050618Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} {"msg":"1f9d774c-3f75-40d3-b197-8c181095b426 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726130988Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726150227Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"0","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726178237Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"0","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726192427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"1","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726215017Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"1","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726241537Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"2","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726268126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"2","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726293756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726319656Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726342856Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"ds_connection connected from Ok(127.0.0.1:53756)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726365806Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55154)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726390776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56805)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.726416716Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.72711406Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"2","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.72716163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"0","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.72719885Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"1","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.727847985Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"0","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.727887394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"1","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.727917934Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"2","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.72843295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.72853176Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[0]R flush_numbers[0..12]: [264, 264, 261, 266, 266, 264, 258, 264, 263, 264, 260, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728601839Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[0]R generation[0..12]: [159, 159, 157, 161, 161, 159, 155, 159, 158, 159, 156, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728622749Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728637959Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[1]R flush_numbers[0..12]: [264, 264, 261, 266, 266, 264, 258, 264, 263, 264, 260, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728652039Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[1]R generation[0..12]: [159, 159, 157, 161, 161, 159, 155, 159, 158, 159, 156, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728665929Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728679568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[2]R flush_numbers[0..12]: [264, 264, 261, 266, 266, 264, 258, 264, 263, 264, 260, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728693028Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[2]R generation[0..12]: [159, 159, 157, 161, 161, 159, 155, 159, 158, 159, 156, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728709828Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728723808Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728736558Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728749668Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728762358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728776238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"":"downstairs","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728789568Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728801398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"0","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728814447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"1","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728827318Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"client":"2","":"downstairs","session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728840187Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728852217Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"1f9d774c-3f75-40d3-b197-8c181095b426 is now active with session: 87ac302a-d1b3-4a5a-ba83-d98adaa4c67d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728865087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} Wait for a query_work_queue command to finish before sending IO {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728882407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840,"session_id":"87ac302a-d1b3-4a5a-ba83-d98adaa4c67d"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728894587Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} {"msg":"Activated sub_volume 1f9d774c-3f75-40d3-b197-8c181095b426","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:11.728905737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 4096 data: 14 02/30 Write block 99 len 4096 data: 11 03/30 Write block 383 len 36864 data: 12 15 19 20 15 15 18 16 12 04/30 Write block 476 len 40960 data: 8 7 8 11 9 10 16 18 18 18 05/30 Read block 159 len 8192 06/30 Write block 76 len 12288 data: 15 14 15 07/30 Read block 506 len 4096 08/30 Flush 09/30 Flush 10/30 Write block 389 len 24576 data: 19 17 13 11 9 7 11/30 Read block 51 len 24576 12/30 Write block 537 len 12288 data: 7 9 11 13/30 Read block 81 len 40960 14/30 Write block 288 len 32768 data: 17 16 18 20 21 21 18 16 15/30 Read block 48 len 28672 16/30 Read block 358 len 20480 17/30 Write block 38 len 32768 data: 21 21 16 16 12 12 10 8 18/30 Read block 339 len 8192 19/30 Read block 254 len 40960 20/30 Read block 199 len 36864 21/30 Write block 285 len 24576 data: 15 15 17 18 17 19 22/30 Write block 418 len 40960 data: 9 12 12 14 13 14 13 13 12 12 23/30 Read block 256 len 8192 24/30 Write block 88 len 16384 data: 11 12 15 13 25/30 Read block 2 len 40960 26/30 Write block 20 len 8192 data: 14 17 27/30 Read block 270 len 20480 28/30 Read block 590 len 40960 29/30 Write block 380 len 12288 data: 10 12 11 30/30 Read block 489 len 16384 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 6 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 19 21 19 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:11.749485053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1840} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 264 264 264 F F F 1 020-039 159 159 159 264 264 264 T T T 2 040-059 157 157 157 261 261 261 T T T 3 060-079 162 162 162 267 267 267 F F F 4 080-099 162 162 162 267 267 267 T T T 5 100-119 159 159 159 264 264 264 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 156 156 156 260 260 260 F F F 11 220-239 161 161 161 266 266 266 F F F 12 240-259 158 158 158 262 262 262 F F F 13 260-279 159 159 159 264 264 264 F F F 14 280-299 161 161 161 266 266 266 T T T 15 300-319 161 161 161 266 266 266 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 162 162 162 267 267 267 T T T 20 400-419 161 161 161 266 266 266 T T T 21 420-439 155 155 155 258 258 258 T T T 22 440-459 161 161 161 266 266 266 F F F 23 460-479 162 162 162 267 267 267 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 161 161 161 266 266 266 F F F 26 520-539 161 161 161 266 266 266 T T T 27 540-559 161 161 161 266 266 266 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 160 160 160 265 265 265 F F F Max gen: 162, Max flush: 267 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 264 264 264 F F F 1 020-039 163 163 163 268 268 268 F F F 2 040-059 163 163 163 268 268 268 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 163 163 163 268 268 268 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 155 155 155 258 258 258 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 156 156 156 260 260 260 F F F 11 220-239 161 161 161 266 266 266 F F F 12 240-259 158 158 158 262 262 262 F F F 13 260-279 159 159 159 264 264 264 F F F 14 280-299 163 163 163 268 268 268 F F F 15 300-319 161 161 161 266 266 266 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 162 162 162 267 267 267 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 163 163 163 268 268 268 F F F 21 420-439 163 163 163 268 268 268 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 162 162 162 267 267 267 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 161 161 161 266 266 266 F F F 26 520-539 163 163 163 268 268 268 F F F 27 540-559 161 161 161 266 266 266 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 160 160 160 265 265 265 F F F Max gen: 163, Max flush: 268 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.005085724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.049582551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.050121937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.050165547Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.050183927Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.050201067Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} {"msg":"Crucible bbfc4e7e-0c71-4031-8414-5fe4e10bb620 has session id: 2447da50-be6f-4b82-95ee-e3fc7891ad2a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.050746843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Upstairs opts: Upstairs UUID: bbfc4e7e-0c71-4031-8414-5fe4e10bb620, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.050788862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Crucible stats registered with UUID: bbfc4e7e-0c71-4031-8414-5fe4e10bb620","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.050813242Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:26.050828422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051211799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051245579Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} {"msg":"bbfc4e7e-0c71-4031-8414-5fe4e10bb620 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051298949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051324418Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"0","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051353508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"0","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051376088Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"1","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051402838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"1","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051428477Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"2","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051461667Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"2","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051488667Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051517117Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051543757Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45951)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051569387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49654)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051596016Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45227)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.051623376Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.052307421Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"1","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.05235601Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"0","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.052925116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"0","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.053072585Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"1","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.053099145Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"2","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.053516162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.053618011Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.05374942Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"2","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[0]R flush_numbers[0..12]: [264, 268, 268, 267, 268, 264, 258, 264, 263, 264, 260, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054254116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[0]R generation[0..12]: [159, 163, 163, 162, 163, 159, 155, 159, 158, 159, 156, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054282136Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054298786Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[1]R flush_numbers[0..12]: [264, 268, 268, 267, 268, 264, 258, 264, 263, 264, 260, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054313136Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[1]R generation[0..12]: [159, 163, 163, 162, 163, 159, 155, 159, 158, 159, 156, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054326816Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054340836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[2]R flush_numbers[0..12]: [264, 268, 268, 267, 268, 264, 258, 264, 263, 264, 260, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054354516Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[2]R generation[0..12]: [159, 163, 163, 162, 163, 159, 155, 159, 158, 159, 156, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054368286Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054382165Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054394975Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054407905Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054487475Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054522044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"":"downstairs","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054554704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054580854Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"0","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054607904Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"1","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054634924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"client":"2","":"downstairs","session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054662903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054692053Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"bbfc4e7e-0c71-4031-8414-5fe4e10bb620 is now active with session: 2447da50-be6f-4b82-95ee-e3fc7891ad2a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054718673Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054733123Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848,"session_id":"2447da50-be6f-4b82-95ee-e3fc7891ad2a"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054746263Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} {"msg":"Activated sub_volume bbfc4e7e-0c71-4031-8414-5fe4e10bb620","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.054757752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 464 len 16384 data: 12 12 10 11 02/30 Write block 296 len 12288 data: 13 12 11 03/30 Read block 192 len 36864 04/30 Read block 201 len 32768 05/30 Write block 205 len 24576 data: 8 10 10 13 13 16 06/30 Read block 9 len 40960 07/30 Write block 127 len 32768 data: 13 13 15 16 21 19 18 17 08/30 Write block 248 len 4096 data: 15 09/30 Flush 10/30 Write block 296 len 28672 data: 14 13 12 12 13 13 11 11/30 Read block 158 len 32768 12/30 Read block 263 len 16384 13/30 Flush 14/30 Write block 341 len 36864 data: 10 10 12 12 12 12 15 11 15 15/30 Read block 364 len 20480 16/30 Write block 4 len 12288 data: 8 11 13 17/30 Read block 99 len 36864 18/30 Read block 191 len 40960 19/30 Read block 216 len 4096 20/30 Read block 198 len 28672 21/30 Read block 468 len 32768 22/30 Read block 353 len 28672 23/30 Read block 113 len 16384 24/30 Read block 274 len 36864 25/30 Flush 26/30 Write block 413 len 40960 data: 13 10 9 9 9 10 13 13 15 14 27/30 Read block 117 len 4096 28/30 Read block 554 len 28672 29/30 Read block 343 len 20480 30/30 Read block 215 len 4096 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:26.076223152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1848} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 271 271 271 F F F 1 020-039 163 163 163 268 268 268 F F F 2 040-059 163 163 163 268 268 268 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 163 163 163 268 268 268 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 164 164 164 269 269 269 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 161 161 161 266 266 266 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 159 159 159 264 264 264 F F F 14 280-299 164 164 164 270 270 270 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 164 164 164 271 271 271 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 163 163 163 268 268 268 T T T 21 420-439 163 163 163 268 268 268 T T T 22 440-459 161 161 161 266 266 266 F F F 23 460-479 164 164 164 269 269 269 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 161 161 161 266 266 266 F F F 26 520-539 163 163 163 268 268 268 F F F 27 540-559 161 161 161 266 266 266 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 160 160 160 265 265 265 F F F Max gen: 164, Max flush: 271 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 271 271 271 F F F 1 020-039 163 163 163 268 268 268 F F F 2 040-059 163 163 163 268 268 268 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 163 163 163 268 268 268 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 164 164 164 269 269 269 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 161 161 161 266 266 266 F F F 12 240-259 164 164 164 269 269 269 F F F 13 260-279 159 159 159 264 264 264 F F F 14 280-299 164 164 164 270 270 270 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 164 164 164 271 271 271 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 165 165 165 272 272 272 F F F 21 420-439 165 165 165 272 272 272 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 164 164 164 269 269 269 F F F 24 480-499 162 162 162 267 267 267 F F F 25 500-519 161 161 161 266 266 266 F F F 26 520-539 163 163 163 268 268 268 F F F 27 540-559 161 161 161 266 266 266 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 160 160 160 265 265 265 F F F Max gen: 165, Max flush: 272 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.294136126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.337907649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.338488024Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.338530574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.338545704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.338560004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} {"msg":"Crucible 27ae0eb2-43cb-44d8-95f1-44bb55541e22 has session id: c183502e-d6ba-4043-b261-e345fe3cf23b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.33906681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Upstairs opts: Upstairs UUID: 27ae0eb2-43cb-44d8-95f1-44bb55541e22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339110289Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Crucible stats registered with UUID: 27ae0eb2-43cb-44d8-95f1-44bb55541e22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339130889Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:30.339144369Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339601096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339631416Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} {"msg":"27ae0eb2-43cb-44d8-95f1-44bb55541e22 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339703935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339734475Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"0","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339765155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"0","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339779864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"1","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339803364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"1","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339829724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"2","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339854444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"2","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339873754Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339893814Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339918423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64550)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.339962703Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61356)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.340012683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44922)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.340042582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.340760377Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"1","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.340788687Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"0","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.341417342Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"1","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.341454002Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"0","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.342068758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.342102797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.342768192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"2","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343267838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"2","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[0]R flush_numbers[0..12]: [271, 268, 268, 267, 268, 264, 269, 264, 263, 264, 269, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343797354Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[0]R generation[0..12]: [164, 163, 163, 162, 163, 159, 164, 159, 158, 159, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343818214Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343832994Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[1]R flush_numbers[0..12]: [271, 268, 268, 267, 268, 264, 269, 264, 263, 264, 269, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343846764Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[1]R generation[0..12]: [164, 163, 163, 162, 163, 159, 164, 159, 158, 159, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343860404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343874254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[2]R flush_numbers[0..12]: [271, 268, 268, 267, 268, 264, 269, 264, 263, 264, 269, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343887824Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[2]R generation[0..12]: [164, 163, 163, 162, 163, 159, 164, 159, 158, 159, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343901534Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343918573Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343931773Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343944833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343957663Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":Wait for a query_work_queue command to finish before sending IO "All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343976573Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"":"downstairs","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.343989513Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344014013Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"0","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344027793Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"1","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344040843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"client":"2","":"downstairs","session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344054143Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344066383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"27ae0eb2-43cb-44d8-95f1-44bb55541e22 is now active with session: c183502e-d6ba-4043-b261-e345fe3cf23b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344078992Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344090752Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856,"session_id":"c183502e-d6ba-4043-b261-e345fe3cf23b"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344102662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} {"msg":"Activated sub_volume 27ae0eb2-43cb-44d8-95f1-44bb55541e22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:30.344113662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 43 len 36864 data: 13 11 9 7 9 10 11 8 9 02/30 Read block 68 len 16384 03/30 Read block 555 len 40960 04/30 Read block 341 len 8192 05/30 Read block 13 len 20480 06/30 Write block 222 len 12288 data: 12 12 11 07/30 Read block 408 len 16384 08/30 Read block 569 len 24576 09/30 Read block 122 len 4096 10/30 Read block 182 len 24576 11/30 Write block 499 len 32768 data: 15 14 14 10 11 11 11 12 12/30 Read block 480 len 24576 13/30 Write block 348 len 40960 data: 12 16 14 20 19 19 21 16 17 16 14/30 Flush 15/30 Write block 357 len 32768 data: 17 14 14 11 10 8 8 10 16/30 Write block 82 len 8192 data: 11 10 17/30 Flush 18/30 Write block 277 len 36864 data: 7 10 12 13 14 13 12 14 16 19/30 Flush 20/30 Write block 36 len 16384 data: 21 19 22 22 21/30 Write block 410 len 16384 data: 12 11 11 14 22/30 Read block 563 len 28672 23/30 Read block 41 len 36864 24/30 Read block 92 len 12288 25/30 Write block 232 len 40960 data: 9 8 9 12 11 10 11 12 11 12 26/30 Read block 295 len 40960 27/30 Write block 510 len 32768 data: 14 14 11 12 8 10 11 14 28/30 Read block 215 len 16384 29/30 Read block 64 len 4096 30/30 Read block 504 len 8192 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:30.366049218Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 271 271 271 F F F 1 020-039 163 163 163 268 268 268 T T T 2 040-059 166 166 166 273 273 273 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 164 164 164 269 269 269 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 166 166 166 273 273 273 T T T 12 240-259 164 164 164 269 269 269 T T T 13 260-279 166 166 166 275 275 275 F F F 14 280-299 166 166 166 275 275 275 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 166 166 166 274 274 274 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 165 165 165 272 272 272 T T T 21 420-439 165 165 165 272 272 272 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 164 164 164 269 269 269 F F F 24 480-499 166 166 166 273 273 273 F F F 25 500-519 166 166 166 273 273 273 T T T 26 520-539 163 163 163 268 268 268 F F F 27 540-559 161 161 161 266 266 266 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 160 160 160 265 265 265 F F F Max gen: 166, Max flush: 275 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 271 271 271 F F F 1 020-039 167 167 167 276 276 276 F F F 2 040-059 166 166 166 273 273 273 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 164 164 164 269 269 269 F F F 7 140-159 159 159 159 264 264 264 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 167 167 167 276 276 276 F F F 12 240-259 167 167 167 276 276 276 F F F 13 260-279 166 166 166 275 275 275 F F F 14 280-299 166 166 166 275 275 275 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 166 166 166 274 274 274 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 167 167 167 276 276 276 F F F 21 420-439 165 165 165 272 272 272 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 164 164 164 269 269 269 F F F 24 480-499 166 166 166 273 273 273 F F F 25 500-519 167 167 167 276 276 276 F F F 26 520-539 163 163 163 268 268 268 F F F 27 540-559 161 161 161 266 266 266 F F F 28 560-579 161 161 161 266 266 266 F F F 29 580-599 160 160 160 265 265 265 F F F Max gen: 167, Max flush: 276 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.602411816Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.646451666Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.647047102Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.647090491Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.647105081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.647119391Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} {"msg":"Crucible 73be8c88-471b-4e7a-8a0d-40cd0263eab8 has session id: 9b4bf250-b9e2-4cb7-8378-9b489dc37975","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.647509428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Upstairs opts: Upstairs UUID: 73be8c88-471b-4e7a-8a0d-40cd0263eab8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.647589988Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Crucible stats registered with UUID: 73be8c88-471b-4e7a-8a0d-40cd0263eab8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.647614747Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:34.647637017Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648184533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648216523Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} {"msg":"73be8c88-471b-4e7a-8a0d-40cd0263eab8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648305992Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648333682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"0","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648373742Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"0","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648390512Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"1","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648405002Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"1","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648418132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"2","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648431861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"2","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648444531Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648460781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648482921Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"ds_connection connected from Ok(127.0.0.1:49885)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.648514161Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"ds_connection connected from Ok(127.0.0.1:53904)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.64863712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"ds_connection connected from Ok(127.0.0.1:42436)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.64867807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.649380524Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"2","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.649420864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"0","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.649451954Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"1","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.64992163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"2","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.65000811Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"1","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650037059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"0","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650432857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650593195Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[0]R flush_numbers[0..12]: [271, 276, 273, 267, 274, 264, 269, 264, 263, 264, 269, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650662845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[0]R generation[0..12]: [164, 167, 166, 162, 166, 159, 164, 159, 158, 159, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650684115Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650702574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[1]R flush_numbers[0..12]: [271, 276, 273, 267, 274, 264, 269, 264, 263, 264, 269, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650716934Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[1]R generation[0..12]: [164, 167, 166, 162, 166, 159, 164, 159, 158, 159, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650730444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650744614Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[2]R flush_numbers[0..12]: [271, 276, 273, 267, 274, 264, 269, 264, 263, 264, 269, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650758734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[2]R generation[0..12]: [164, 167, 166, 162, 166, 159, 164, 159, 158, 159, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650772164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650785724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650798294Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650811463Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650824163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650836833Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"":"downstairs","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650849753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650861593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"0","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650874533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"1","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650887183Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"client":"2","":"downstairs","session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650901683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650913963Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"73be8c88-471b-4e7a-8a0d-40cd0263eab8 is now active with session: 9b4bf250-b9e2-4cb7-8378-9b489dc37975","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:34.650926463Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650945122Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864,"session_id":"9b4bf250-b9e2-4cb7-8378-9b489dc37975"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650957293Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} {"msg":"Activated sub_volume 73be8c88-471b-4e7a-8a0d-40cd0263eab8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:34.650968462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 241 len 20480 data: 13 12 12 12 10 02/30 Write block 366 len 20480 data: 7 9 9 9 10 03/30 Read block 163 len 32768 04/30 Write block 123 len 36864 data: 16 19 17 11 14 14 16 17 22 05/30 Write block 474 len 36864 data: 9 8 9 8 9 12 10 11 17 06/30 Write block 10 len 16384 data: 17 19 16 15 07/30 Write block 150 len 24576 data: 15 15 17 15 13 11 08/30 Write block 513 len 36864 data: 13 9 11 12 15 15 18 18 20 09/30 Read block 18 len 36864 10/30 Read block 583 len 36864 11/30 Read block 400 len 4096 12/30 Flush 13/30 Write block 294 len 16384 data: 19 17 15 14 14/30 Write block 465 len 8192 data: 13 11 15/30 Write block 543 len 32768 data: 11 14 16 17 21 18 19 14 16/30 Read block 92 len 8192 17/30 Write block 484 len 12288 data: 19 19 19 18/30 Read block 49 len 24576 19/30 Write block 566 len 24576 data: 10 8 10 10 12 11 20/30 Write block 510 len 4096 data: 15 21/30 Read block 317 len 36864 22/30 Write block 136 len 16384 data: 15 14 13 12 23/30 Write block 50 len 40960 data: 9 10 9 10 10 12 12 10 10 10 24/30 Write block 546 len 20480 data: 18 22 19 20 15 25/30 Read block 140 len 20480 26/30 Flush 27/30 Write block 592 len 16384 data: 11 11 10 12 28/30 Write block 485 len 24576 data: 20 20 18 16 19 18 29/30 Read block 494 len 24576 30/30 Read block 133 len 4096 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:34.66996004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1864} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 277 277 277 F F F 1 020-039 167 167 167 276 276 276 F F F 2 040-059 168 168 168 278 278 278 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 168 168 168 278 278 278 F F F 7 140-159 168 168 168 277 277 277 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 167 167 167 276 276 276 F F F 12 240-259 168 168 168 277 277 277 F F F 13 260-279 166 166 166 275 275 275 F F F 14 280-299 168 168 168 278 278 278 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 168 168 168 277 277 277 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 167 167 167 276 276 276 F F F 21 420-439 165 165 165 272 272 272 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 168 168 168 278 278 278 F F F 24 480-499 168 168 168 278 278 278 T T T 25 500-519 168 168 168 278 278 278 F F F 26 520-539 168 168 168 277 277 277 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 160 160 160 265 265 265 T T T Max gen: 168, Max flush: 278 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 277 277 277 F F F 1 020-039 167 167 167 276 276 276 F F F 2 040-059 168 168 168 278 278 278 F F F 3 060-079 162 162 162 267 267 267 F F F 4 080-099 166 166 166 274 274 274 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 168 168 168 278 278 278 F F F 7 140-159 168 168 168 277 277 277 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 159 159 159 264 264 264 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 167 167 167 276 276 276 F F F 12 240-259 168 168 168 277 277 277 F F F 13 260-279 166 166 166 275 275 275 F F F 14 280-299 168 168 168 278 278 278 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 168 168 168 277 277 277 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 167 167 167 276 276 276 F F F 21 420-439 165 165 165 272 272 272 F F F 22 440-459 161 161 161 266 266 266 F F F 23 460-479 168 168 168 278 278 278 F F F 24 480-499 169 169 169 279 279 279 F F F 25 500-519 168 168 168 278 278 278 F F F 26 520-539 168 168 168 277 277 277 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 169 169 169 279 279 279 F F F Max gen: 169, Max flush: 279 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.888046754Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.931846386Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.932506911Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.932548831Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.932563351Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.932577991Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} {"msg":"Crucible d9e1e1a6-42ef-490f-af1d-1ca4781664a7 has session id: 93f8065d-3243-4bf5-99e4-091a0e36d175","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.933186156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Upstairs opts: Upstairs UUID: d9e1e1a6-42ef-490f-af1d-1ca4781664a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.933225986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Crucible stats registered with UUID: d9e1e1a6-42ef-490f-af1d-1ca4781664a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.933249576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:38.933271426Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.933808191Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.933834961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} {"msg":"d9e1e1a6-42ef-490f-af1d-1ca4781664a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.933910151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93392827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"0","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93395838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"0","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.9339768Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"1","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.9339982Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"1","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93402291Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"2","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93405034Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"2","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934078789Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934108219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934138639Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"ds_connection connected from Ok(127.0.0.1:35114)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934166369Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"ds_connection connected from Ok(127.0.0.1:65178)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934193908Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"ds_connection connected from Ok(127.0.0.1:38826)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934220368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934855443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"0","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934888013Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"1","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.934904433Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"2","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.935555938Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"1","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.935589598Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"2","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.935611438Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"0","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936157834Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936264923Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[0]R flush_numbers[0..12]: [277, 276, 278, 267, 274, 264, 278, 277, 263, 264, 269, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936312253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[0]R generation[0..12]: [168, 167, 168, 162, 166, 159, 168, 168, 158, 159, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936331683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936346623Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[1]R flush_numbers[0..12]: [277, 276, 278, 267, 274, 264, 278, 277, 263, 264, 269, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936360572Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[1]R generation[0..12]: [168, 167, 168, 162, 166, 159, 168, 168, 158, 159, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936374132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936387882Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[2]R flush_numbers[0..12]: [277, 276, 278, 267, 274, 264, 278, 277, 263, 264, 269, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936401592Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[2]R generation[0..12]: [168, 167, 168, 162, 166, 159, 168, 168, 158, 159, 164, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936415072Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936428892Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936449682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:38.936478442Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936509811Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936531611Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"":"downstairs","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936545481Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936557401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"0","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936570541Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"1","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.936583311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"client":"2","":"downstairs","session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93659633Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93660851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"d9e1e1a6-42ef-490f-af1d-1ca4781664a7 is now active with session: 93f8065d-3243-4bf5-99e4-091a0e36d175","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93662128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93663311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872,"session_id":"93f8065d-3243-4bf5-99e4-091a0e36d175"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93664521Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} {"msg":"Activated sub_volume d9e1e1a6-42ef-490f-af1d-1ca4781664a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:38.93665715Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 184 len 12288 data: 17 15 14 02/30 Read block 266 len 36864 03/30 Write block 221 len 40960 data: 14 13 13 12 12 12 13 14 12 13 04/30 Read block 372 len 32768 05/30 Write block 40 len 12288 data: 17 17 13 06/30 Read block 171 len 36864 07/30 Write block 378 len 4096 data: 13 08/30 Read block 291 len 4096 09/30 Read block 11 len 28672 10/30 Write block 233 len 40960 data: 9 10 13 12 11 12 13 12 14 13 11/30 Read block 138 len 12288 12/30 Write block 34 len 36864 data: 17 18 22 20 23 23 18 18 14 13/30 Read block 216 len 16384 14/30 Write block 440 len 16384 data: 10 8 10 12 15/30 Read block 120 len 28672 16/30 Write block 521 len 8192 data: 21 20 17/30 Write block 276 len 40960 data: 10 8 11 13 14 15 14 13 15 17 18/30 Flush 19/30 Write block 404 len 40960 data: 15 14 12 12 11 13 13 12 12 15 20/30 Write block 81 len 8192 data: 11 12 21/30 Read block 220 len 24576 22/30 Write block 70 len 8192 data: 14 13 23/30 Read block 318 len 28672 24/30 Write block 18 len 32768 data: 15 15 15 18 17 20 17 14 25/30 Read block 376 len 16384 26/30 Read block 307 len 40960 27/30 Read block 310 len 40960 28/30 Write block 136 len 24576 data: 16 15 14 13 14 12 29/30 Write block 32 len 36864 data: 14 15 18 19 23 21 24 24 19 30/30 Write block 262 len 28672 data: 10 13 10 14 12 12 12 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:38.957853751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1872} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 277 277 277 T T T 1 020-039 170 170 170 280 280 280 T T T 2 040-059 170 170 170 280 280 280 T T T 3 060-079 162 162 162 267 267 267 T T T 4 080-099 166 166 166 274 274 274 T T T 5 100-119 159 159 159 264 264 264 F F F 6 120-139 168 168 168 278 278 278 T T T 7 140-159 168 168 168 277 277 277 T T T 8 160-179 158 158 158 263 263 263 F F F 9 180-199 170 170 170 280 280 280 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 170 170 170 280 280 280 F F F 12 240-259 170 170 170 280 280 280 F F F 13 260-279 170 170 170 280 280 280 T T T 14 280-299 170 170 170 280 280 280 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 167 167 167 276 276 276 T T T 21 420-439 165 165 165 272 272 272 F F F 22 440-459 170 170 170 280 280 280 F F F 23 460-479 168 168 168 278 278 278 F F F 24 480-499 169 169 169 279 279 279 F F F 25 500-519 168 168 168 278 278 278 F F F 26 520-539 170 170 170 280 280 280 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 169 169 169 279 279 279 F F F Max gen: 170, Max flush: 280 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 281 281 281 F F F 1 020-039 171 171 171 281 281 281 F F F 2 040-059 171 171 171 281 281 281 F F F 3 060-079 171 171 171 281 281 281 F F F 4 080-099 171 171 171 281 281 281 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 171 171 171 281 281 281 F F F 7 140-159 171 171 171 281 281 281 F F F 8 160-179 158 158 158 263 263 263 F F F 9 180-199 170 170 170 280 280 280 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 170 170 170 280 280 280 F F F 12 240-259 170 170 170 280 280 280 F F F 13 260-279 171 171 171 281 281 281 F F F 14 280-299 170 170 170 280 280 280 F F F 15 300-319 164 164 164 270 270 270 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 171 171 171 281 281 281 F F F 21 420-439 165 165 165 272 272 272 F F F 22 440-459 170 170 170 280 280 280 F F F 23 460-479 168 168 168 278 278 278 F F F 24 480-499 169 169 169 279 279 279 F F F 25 500-519 168 168 168 278 278 278 F F F 26 520-539 170 170 170 280 280 280 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 169 169 169 279 279 279 F F F Max gen: 171, Max flush: 281 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.213923271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.257674233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.258275219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.258319799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.258335238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.258349918Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} {"msg":"Crucible 94c46fdd-7245-4f00-a567-f0d751d0a935 has session id: f8270d34-2b63-482a-b5cc-bf49fa31e8d8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.258951024Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Upstairs opts: Upstairs UUID: 94c46fdd-7245-4f00-a567-f0d751d0a935, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.258998154Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Crucible stats registered with UUID: 94c46fdd-7245-4f00-a567-f0d751d0a935","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259027163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:43.259048323Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.25951044Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259548639Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} {"msg":"94c46fdd-7245-4f00-a567-f0d751d0a935 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259624209Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259677258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"0","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259745608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"0","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259780388Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"1","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259821337Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"1","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259854857Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259889577Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259923516Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259956896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.259987666Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:43.260028096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:43.260067935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:43.260133785Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.260170325Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:46470)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.260216864Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41128)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.260251404Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.260672661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"0","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.26086264Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"1","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.261244387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"0","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.261353756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"1","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.261773483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:43.261945411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.260772219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48303)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.260903708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.262467526Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263000832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 281, 281, 281, 264, 281, 281, 263, 280, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263644218Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[0]R generation[0..12]: [171, 171, 171, 171, 171, 159, 171, 171, 158, 170, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263666067Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263681027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 281, 281, 281, 264, 281, 281, 263, 280, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263695287Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[1]R generation[0..12]: [171, 171, 171, 171, 171, 159, 171, 171, 158, 170, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263708817Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263722437Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 281, 281, 281, 264, 281, 281, 263, 280, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263735777Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[2]R generation[0..12]: [171, 171, 171, 171, 171, 159, 171, 171, 158, 170, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263749197Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263762966Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263775706Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263788726Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263801416Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263818316Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"":"downstairs","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263831826Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263843716Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"0","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263857086Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"1","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263870296Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"client":"2","":"downstairs","session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"All required reconciliation work is completed"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263891006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263903636Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"94c46fdd-7245-4f00-a567-f0d751d0a935 is now active with session: f8270d34-2b63-482a-b5cc-bf49fa31e8d8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263916425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263928405Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880,"session_id":"f8270d34-2b63-482a-b5cc-bf49fa31e8d8"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263940505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} {"msg":"Activated sub_volume 94c46fdd-7245-4f00-a567-f0d751d0a935","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:53.263951635Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 183 len 28672 data: 15 18 16 15 14 10 11 03/30 Flush 04/30 Write block 591 len 4096 data: 10 05/30 Read block 121 len 12288 06/30 Write block 291 len 32768 data: 21 22 22 20 18 16 15 13 07/30 Write block 305 len 8192 data: 9 8 08/30 Flush 09/30 Flush 10/30 Write block 305 len 12288 data: 10 9 9 11/30 Write block 409 len 8192 data: 14 14 12/30 Flush 13/30 Write block 490 len 16384 data: 19 19 22 21 14/30 Read block 196 len 4096 15/30 Read block 407 len 28672 16/30 Read block 220 len 20480 17/30 Read block 276 len 4096 18/30 Flush 19/30 Write block 418 len 24576 data: 11 14 14 16 15 15 20/30 Read block 103 len 20480 21/30 Flush 22/30 Write block 266 len 40960 data: 13 13 13 12 13 13 13 13 11 15 23/30 Read block 279 len 20480 24/30 Read block 481 len 20480 25/30 Flush 26/30 Write block 169 len 28672 data: 15 16 13 14 16 15 13 27/30 Write block 6 len 36864 data: 14 15 14 17 18 20 17 16 18 28/30 Read block 141 len 28672 29/30 Read block 348 len 28672 30/30 Write block 128 len 16384 data: 15 17 18 23 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:53.283977885Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1880} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 281 281 281 T T T 1 020-039 171 171 171 281 281 281 F F F 2 040-059 171 171 171 281 281 281 F F F 3 060-079 171 171 171 281 281 281 F F F 4 080-099 171 171 171 281 281 281 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 171 171 171 281 281 281 T T T 7 140-159 171 171 171 281 281 281 F F F 8 160-179 158 158 158 263 263 263 T T T 9 180-199 172 172 172 283 283 283 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 170 170 170 280 280 280 F F F 12 240-259 170 170 170 280 280 280 F F F 13 260-279 172 172 172 289 289 289 F F F 14 280-299 172 172 172 284 284 284 F F F 15 300-319 172 172 172 286 286 286 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 172 172 172 288 288 288 F F F 22 440-459 170 170 170 280 280 280 F F F 23 460-479 168 168 168 278 278 278 F F F 24 480-499 172 172 172 287 287 287 F F F 25 500-519 168 168 168 278 278 278 F F F 26 520-539 170 170 170 280 280 280 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 172, Max flush: 289 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 290 290 290 F F F 1 020-039 171 171 171 281 281 281 F F F 2 040-059 171 171 171 281 281 281 F F F 3 060-079 171 171 171 281 281 281 F F F 4 080-099 171 171 171 281 281 281 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 173 173 173 290 290 290 F F F 7 140-159 171 171 171 281 281 281 F F F 8 160-179 173 173 173 290 290 290 F F F 9 180-199 172 172 172 283 283 283 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 170 170 170 280 280 280 F F F 12 240-259 170 170 170 280 280 280 F F F 13 260-279 172 172 172 289 289 289 F F F 14 280-299 172 172 172 284 284 284 F F F 15 300-319 172 172 172 286 286 286 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 166 166 166 274 274 274 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 172 172 172 288 288 288 F F F 22 440-459 170 170 170 280 280 280 F F F 23 460-479 168 168 168 278 278 278 F F F 24 480-499 172 172 172 287 287 287 F F F 25 500-519 168 168 168 278 278 278 F F F 26 520-539 170 170 170 280 280 280 F F F 27 540-559 168 168 168 278 278 278 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 173, Max flush: 290 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.512861708Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:57.555004503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.55536608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.586207309Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.586769135Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.586805125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.586821105Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.586838865Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} {"msg":"Crucible 9eb4802d-2bba-4d8c-90ec-1f8c43fefea8 has session id: d0c4edce-171a-477e-86d4-a1e1888efc3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.587387401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Upstairs opts: Upstairs UUID: 9eb4802d-2bba-4d8c-90ec-1f8c43fefea8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.58743802Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Crucible stats registered with UUID: 9eb4802d-2bba-4d8c-90ec-1f8c43fefea8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.58747043Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:57.58750324Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.587847167Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.587881247Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} {"msg":"9eb4802d-2bba-4d8c-90ec-1f8c43fefea8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.587934876Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.587962766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"0","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588001386Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"0","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588028766Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"1","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588056525Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"1","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588086735Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"2","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588116615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"2","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588143725Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588172005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588200164Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64168)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588228704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50164)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588274574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62257)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588359633Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588937749Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"2","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.588967179Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"1","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.589070608Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"0","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.589605564Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"1","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.589635274Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"2","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.589651504Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"0","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590212269Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590310679Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[0]R flush_numbers[0..12]: [290, 281, 281, 281, 281, 264, 290, 281, 290, 283, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590383298Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[0]R generation[0..12]: [173, 171, 171, 171, 171, 159, 173, 171, 173, 172, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590404898Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590420428Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[1]R flush_numbers[0..12]: [290, 281, 281, 281, 281, 264, 290, 281, 290, 283, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590434378Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[1]R generation[0..12]: [173, 171, 171, 171, 171, 159, 173, 171, 173, 172, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590448387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590462538Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[2]R flush_numbers[0..12]: [290, 281, 281, 281, 281, 264, 290, 281, 290, 283, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590480077Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[2]R generation[0..12]: [173, 171, 171, 171, 171, 159, 173, 171, 173, 172, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590494527Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590508587Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590521537Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590534597Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590547447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590568027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"":"downstairs","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590581146Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590593126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"0","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590606246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"1","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590619126Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"client":"2","":"downstairs","session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590632316Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590644496Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"9eb4802d-2bba-4d8c-90ec-1f8c43fefea8 is now active with session: d0c4edce-171a-477e-86d4-a1e1888efc3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590657316Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590669166Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888,"session_id":"d0c4edce-171a-477e-86d4-a1e1888efc3a"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590683756Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} {"msg":"Activated sub_volume 9eb4802d-2bba-4d8c-90ec-1f8c43fefea8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:57.590694986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 39 len 40960 data: 25 20 19 15 14 12 10 8 10 11 02/30 Read block 539 len 40960 03/30 Read block 532 len 36864 04/30 Write block 350 len 24576 data: 15 21 20 20 22 17 05/30 Read block 61 len 28672 06/30 Read block 395 len 36864 07/30 Read block 357 len 28672 08/30 Read block 297 len 12288 09/30 Write block 197 len 8192 data: 9 7 10/30 Write block 548 len 36864 data: 20 21 16 9 13 14 13 17 18 11/30 Read block 78 len 12288 12/30 Write block 57 len 12288 data: 11 11 11 13/30 Read block 104 len 16384 14/30 Write block 478 len 24576 data: 10 13 11 12 18 19 15/30 Write block 246 len 12288 data: 10 11 16 16/30 Read block 112 len 4096 17/30 Write block 528 len 24576 data: 10 11 9 9 8 9 18/30 Read block 460 len 32768 19/30 Read block 93 len 4096 20/30 Write block 59 len 32768 data: 12 9 9 6 8 10 12 15 21/30 Read block 389 len 36864 22/30 Read block 348 len 36864 23/30 Read block 7 len 16384 24/30 Write block 452 len 8192 data: 7 8 25/30 Write block 515 len 36864 data: 12 13 16 16 19 19 22 21 13 26/30 Flush 27/30 Write block 504 len 4096 data: 12 28/30 Read block 43 len 24576 29/30 Read block 190 len 32768 30/30 Read block 63 len 24576 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:57.61280335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1888} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 290 290 290 F F F 1 020-039 174 174 174 291 291 291 F F F 2 040-059 174 174 174 291 291 291 F F F 3 060-079 174 174 174 291 291 291 F F F 4 080-099 171 171 171 281 281 281 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 173 173 173 290 290 290 F F F 7 140-159 171 171 171 281 281 281 F F F 8 160-179 173 173 173 290 290 290 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 170 170 170 280 280 280 F F F 12 240-259 174 174 174 291 291 291 F F F 13 260-279 172 172 172 289 289 289 F F F 14 280-299 172 172 172 284 284 284 F F F 15 300-319 172 172 172 286 286 286 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 172 172 172 288 288 288 F F F 22 440-459 174 174 174 291 291 291 F F F 23 460-479 174 174 174 291 291 291 F F F 24 480-499 174 174 174 291 291 291 F F F 25 500-519 174 174 174 291 291 291 T T T 26 520-539 174 174 174 291 291 291 F F F 27 540-559 174 174 174 291 291 291 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 174, Max flush: 291 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 290 290 290 F F F 1 020-039 174 174 174 291 291 291 F F F 2 040-059 174 174 174 291 291 291 F F F 3 060-079 174 174 174 291 291 291 F F F 4 080-099 171 171 171 281 281 281 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 173 173 173 290 290 290 F F F 7 140-159 171 171 171 281 281 281 F F F 8 160-179 173 173 173 290 290 290 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 164 164 164 269 269 269 F F F 11 220-239 170 170 170 280 280 280 F F F 12 240-259 174 174 174 291 291 291 F F F 13 260-279 172 172 172 289 289 289 F F F 14 280-299 172 172 172 284 284 284 F F F 15 300-319 172 172 172 286 286 286 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 172 172 172 288 288 288 F F F 22 440-459 174 174 174 291 291 291 F F F 23 460-479 174 174 174 291 291 291 F F F 24 480-499 174 174 174 291 291 291 F F F 25 500-519 175 175 175 292 292 292 F F F 26 520-539 174 174 174 291 291 291 F F F 27 540-559 174 174 174 291 291 291 F F F 28 560-579 168 168 168 278 278 278 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 175, Max flush: 292 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.828383912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.872056485Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87263444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87268131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87269768Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87271649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} {"msg":"Crucible 5ea8b6ee-a52c-42fa-bd04-b42a39c83344 has session id: 39e2f3ba-f8b8-488a-95ec-e9754e033c5a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.873302386Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Upstairs opts: Upstairs UUID: 5ea8b6ee-a52c-42fa-bd04-b42a39c83344, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.873331355Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Crucible stats registered with UUID: 5ea8b6ee-a52c-42fa-bd04-b42a39c83344","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.873347435Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:01.873360295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.873836511Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.873862301Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} {"msg":"5ea8b6ee-a52c-42fa-bd04-b42a39c83344 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.873934121Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.873951891Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"0","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87397957Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"0","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.8740152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"1","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87404799Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"1","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87407586Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"2","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.87410114Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"2","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.874124969Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.874155209Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.874180119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36605)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.874204189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64380)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.874230268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48416)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.874320868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.874846904Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"1","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.875022123Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"0","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.875458649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"1","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.875608858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"0","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.875736237Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"2","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.876027845Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.876142734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.876411492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"2","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[0]R flush_numbers[0..12]: [290, 291, 291, 291, 281, 264, 290, 281, 290, 291, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.876923728Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[0]R generation[0..12]: [173, 174, 174, 174, 171, 159, 173, 171, 173, 174, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.876954238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.876972908Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[1]R flush_numbers[0..12]: [290, 291, 291, 291, 281, 264, 290, 281, 290, 291, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.876990078Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[1]R generation[0..12]: [173, 174, 174, 174, 171, 159, 173, 171, 173, 174, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877018018Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877041838Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[2]R flush_numbers[0..12]: [290, 291, 291, 291, 281, 264, 290, 281, 290, 291, 269, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877059547Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [173, 174, 174, 174, 171, 159, 173, 171, 173, 174, 164, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877082157Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877108297Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877125407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877141557Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877157447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877173297Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"":"downstairs","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877189357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877207246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"0","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877226396Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"1","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877242716Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"client":"2","":"downstairs","session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877258806Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877273866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"5ea8b6ee-a52c-42fa-bd04-b42a39c83344 is now active with session: 39e2f3ba-f8b8-488a-95ec-e9754e033c5a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877289776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877304425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896,"session_id":"39e2f3ba-f8b8-488a-95ec-e9754e033c5a"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877319476Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} {"msg":"Activated sub_volume 5ea8b6ee-a52c-42fa-bd04-b42a39c83344","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:01.877333305Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 78 len 12288 data: 16 14 10 02/30 Write block 147 len 32768 data: 12 15 16 16 16 18 16 14 03/30 Read block 13 len 8192 04/30 Write block 487 len 40960 data: 19 17 20 20 20 23 22 18 16 12 05/30 Write block 222 len 28672 data: 14 14 13 13 13 14 15 06/30 Read block 495 len 16384 07/30 Read block 407 len 16384 08/30 Read block 25 len 28672 09/30 Read block 192 len 20480 10/30 Read block 107 len 28672 11/30 Flush 12/30 Write block 308 len 40960 data: 8 8 8 8 9 8 8 10 12 10 13/30 Read block 582 len 8192 14/30 Read block 83 len 16384 15/30 Write block 130 len 12288 data: 19 24 20 16/30 Write block 86 len 4096 data: 11 17/30 Write block 442 len 40960 data: 11 13 11 9 11 10 10 12 10 8 18/30 Write block 463 len 24576 data: 13 13 14 12 12 12 19/30 Flush 20/30 Write block 4 len 40960 data: 9 12 15 16 15 18 19 21 18 17 21/30 Read block 202 len 4096 22/30 Flush 23/30 Write block 218 len 40960 data: 14 12 12 15 15 15 14 14 14 15 24/30 Read block 355 len 4096 25/30 Read block 155 len 4096 26/30 Write block 574 len 8192 data: 11 13 27/30 Read block 51 len 24576 28/30 Write block 296 len 40960 data: 17 16 14 13 14 14 12 10 10 11 29/30 Write block 165 len 20480 data: 10 16 16 12 16 30/30 Read block 407 len 12288 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 7 8 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:01.897497904Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1896} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 295 295 295 F F F 1 020-039 174 174 174 291 291 291 F F F 2 040-059 174 174 174 291 291 291 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 176 176 176 294 294 294 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 176 176 176 294 294 294 F F F 7 140-159 176 176 176 293 293 293 F F F 8 160-179 173 173 173 290 290 290 T T T 9 180-199 174 174 174 291 291 291 F F F 10 200-219 164 164 164 269 269 269 T T T 11 220-239 176 176 176 293 293 293 T T T 12 240-259 174 174 174 291 291 291 F F F 13 260-279 172 172 172 289 289 289 F F F 14 280-299 172 172 172 284 284 284 T T T 15 300-319 176 176 176 294 294 294 T T T 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 172 172 172 288 288 288 F F F 22 440-459 176 176 176 294 294 294 F F F 23 460-479 176 176 176 294 294 294 F F F 24 480-499 176 176 176 293 293 293 F F F 25 500-519 175 175 175 292 292 292 F F F 26 520-539 174 174 174 291 291 291 F F F 27 540-559 174 174 174 291 291 291 F F F 28 560-579 168 168 168 278 278 278 T T T 29 580-599 172 172 172 284 284 284 F F F Max gen: 176, Max flush: 295 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 295 295 295 F F F 1 020-039 174 174 174 291 291 291 F F F 2 040-059 174 174 174 291 291 291 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 176 176 176 294 294 294 F F F 5 100-119 159 159 159 264 264 264 F F F 6 120-139 176 176 176 294 294 294 F F F 7 140-159 176 176 176 293 293 293 F F F 8 160-179 177 177 177 296 296 296 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 177 177 177 296 296 296 F F F 11 220-239 177 177 177 296 296 296 F F F 12 240-259 174 174 174 291 291 291 F F F 13 260-279 172 172 172 289 289 289 F F F 14 280-299 177 177 177 296 296 296 F F F 15 300-319 177 177 177 296 296 296 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 172 172 172 288 288 288 F F F 22 440-459 176 176 176 294 294 294 F F F 23 460-479 176 176 176 294 294 294 F F F 24 480-499 176 176 176 293 293 293 F F F 25 500-519 175 175 175 292 292 292 F F F 26 520-539 174 174 174 291 291 291 F F F 27 540-559 174 174 174 291 291 291 F F F 28 560-579 177 177 177 296 296 296 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 177, Max flush: 296 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.140472383Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.185202828Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.185951702Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.185996582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.186012462Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.186026812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} {"msg":"Crucible 11a9e8f6-ab47-4627-8dfe-c3aedf1d14df has session id: 93debf0a-6d52-4449-89fc-8d12cd85e7ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.186483378Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Upstairs opts: Upstairs UUID: 11a9e8f6-ab47-4627-8dfe-c3aedf1d14df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.186522598Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Crucible stats registered with UUID: 11a9e8f6-ab47-4627-8dfe-c3aedf1d14df","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.186546868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:06.186568218Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187007184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187034364Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} {"msg":"11a9e8f6-ab47-4627-8dfe-c3aedf1d14df active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187104444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187126733Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"0","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187155463Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"0","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187169953Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"1","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187184423Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"1","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187197453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"2","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187256663Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"2","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187307412Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187353252Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187391831Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:53630)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187419871Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:34103)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187443831Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:50098)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.18762385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187957387Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"0","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.187986357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"2","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.188415074Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"1","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.188674662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"2","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.188703962Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"0","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189007749Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"1","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189287227Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189389347Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[0]R flush_numbers[0..12]: [295, 291, 291, 293, 294, 264, 294, 293, 296, 291, 296, 296]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189568815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[0]R generation[0..12]: [176, 174, 174, 176, 176, 159, 176, 176, 177, 174, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189598425Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189614175Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[1]R flush_numbers[0..12]: [295, 291, 291, 293, 294, 264, 294, 293, 296, 291, 296, 296]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189628755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[1]R generation[0..12]: [176, 174, 174, 176, 176, 159, 176, 176, 177, 174, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189642534Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189656625Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[2]R flush_numbers[0..12]: [295, 291, 291, 293, 294, 264, 294, 293, 296, 291, 296, 296]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189670914Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[2]R generation[0..12]: [176, 174, 174, 176, 176, 159, 176, 176, 177, 174, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189684564Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189698464Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189715444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7caWait for a query_work_queue command to finish before sending IO "} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189735304Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189748424Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189761434Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"":"downstairs","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189774354Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189786184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"0","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189799233Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"1","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189812113Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"client":"2","":"downstairs","session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189825083Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189837253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"11a9e8f6-ab47-4627-8dfe-c3aedf1d14df is now active with session: 93debf0a-6d52-4449-89fc-8d12cd85e7ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189850003Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189861783Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904,"session_id":"93debf0a-6d52-4449-89fc-8d12cd85e7ca"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189873933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} {"msg":"Activated sub_volume 11a9e8f6-ab47-4627-8dfe-c3aedf1d14df","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:06.189885013Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 156 len 28672 data: 12 10 14 13 10 9 9 02/30 Write block 447 len 20480 data: 11 11 13 11 9 03/30 Flush 04/30 Flush 05/30 Write block 420 len 4096 data: 15 06/30 Flush 07/30 Write block 152 len 16384 data: 19 17 15 12 08/30 Read block 58 len 4096 09/30 Read block 163 len 24576 10/30 Flush 11/30 Write block 99 len 28672 data: 12 12 9 7 7 8 9 12/30 Write block 502 len 8192 data: 11 12 13/30 Write block 265 len 24576 data: 15 14 14 14 13 14 14/30 Read block 304 len 24576 15/30 Read block 419 len 24576 16/30 Write block 235 len 8192 data: 14 13 17/30 Read block 377 len 12288 18/30 Read block 196 len 16384 19/30 Write block 89 len 4096 data: 13 20/30 Write block 203 len 40960 data: 8 8 9 11 11 14 14 17 17 16 21/30 Write block 305 len 24576 data: 12 10 10 9 9 9 22/30 Read block 406 len 8192 23/30 Write block 545 len 4096 data: 17 24/30 Read block 218 len 28672 25/30 Write block 8 len 4096 data: 16 26/30 Read block 559 len 8192 27/30 Write block 283 len 12288 data: 14 16 18 28/30 Read block 54 len 40960 29/30 Write block 561 len 16384 data: 15 12 11 11 30/30 Read block 200 len 4096 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 20 18 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:06.207923278Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1904} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 295 295 295 T T T 1 020-039 174 174 174 291 291 291 F F F 2 040-059 174 174 174 291 291 291 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 176 176 176 294 294 294 T T T 5 100-119 159 159 159 264 264 264 T T T 6 120-139 176 176 176 294 294 294 F F F 7 140-159 178 178 178 300 300 300 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 177 177 177 296 296 296 T T T 11 220-239 177 177 177 296 296 296 T T T 12 240-259 174 174 174 291 291 291 F F F 13 260-279 172 172 172 289 289 289 T T T 14 280-299 177 177 177 296 296 296 T T T 15 300-319 177 177 177 296 296 296 T T T 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 178 178 178 299 299 299 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 176 176 176 294 294 294 F F F 24 480-499 176 176 176 293 293 293 F F F 25 500-519 175 175 175 292 292 292 T T T 26 520-539 174 174 174 291 291 291 F F F 27 540-559 174 174 174 291 291 291 T T T 28 560-579 177 177 177 296 296 296 T T T 29 580-599 172 172 172 284 284 284 F F F Max gen: 178, Max flush: 300 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 301 301 301 F F F 1 020-039 174 174 174 291 291 291 F F F 2 040-059 174 174 174 291 291 291 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 179 179 179 301 301 301 F F F 5 100-119 179 179 179 301 301 301 F F F 6 120-139 176 176 176 294 294 294 F F F 7 140-159 178 178 178 300 300 300 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 F F F 10 200-219 179 179 179 301 301 301 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 174 174 174 291 291 291 F F F 13 260-279 179 179 179 301 301 301 F F F 14 280-299 179 179 179 301 301 301 F F F 15 300-319 179 179 179 301 301 301 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 170 170 170 280 280 280 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 178 178 178 299 299 299 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 176 176 176 294 294 294 F F F 24 480-499 176 176 176 293 293 293 F F F 25 500-519 179 179 179 301 301 301 F F F 26 520-539 174 174 174 291 291 291 F F F 27 540-559 179 179 179 301 301 301 F F F 28 560-579 179 179 179 301 301 301 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 179, Max flush: 301 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.477558706Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.521307368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.522171102Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.522212671Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.522227341Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.522246151Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} {"msg":"Crucible 026af693-7750-42f0-991c-bc59f87c4f26 has session id: 2ff8c933-8da9-467d-b285-5913d079e01b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.522741187Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Upstairs opts: Upstairs UUID: 026af693-7750-42f0-991c-bc59f87c4f26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.522780117Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Crucible stats registered with UUID: 026af693-7750-42f0-991c-bc59f87c4f26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.522803127Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:10.522822177Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523315313Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523346373Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} {"msg":"026af693-7750-42f0-991c-bc59f87c4f26 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523420482Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523447192Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"0","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523477772Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"0","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523492372Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"1","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523515621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"1","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523540941Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"2","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523567401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"2","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523593201Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.523622681Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.52364806Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60696)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.52367755Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50387)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.5237037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41810)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.52373043Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.524454994Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"0","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.524496194Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"1","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.525158459Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"1","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.525198089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"0","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.525790444Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.525816554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.525830594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"2","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.52635439Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"2","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[0]R flush_numbers[0..12]: [301, 291, 291, 293, 301, 301, 294, 300, 297, 291, 301, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526849627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[0]R generation[0..12]: [179, 174, 174, 176, 179, 179, 176, 178, 178, 174, 179, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526875016Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526890876Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[1]R flush_numbers[0..12]: [301, 291, 291, 293, 301, 301, 294, 300, 297, 291, 301, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526906096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[1]R generation[0..12]: [179, 174, 174, 176, 179, 179, 176, 178, 178, 174, 179, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526919986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526933866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[2]R flush_numbers[0..12]: [301, 291, 291, 293, 301, 301, 294, 300, 297, 291, 301, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526947536Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[2]R generation[0..12]: [179, 174, 174, 176, 179, 179, 176, 178, 178, 174, 179, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526961046Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526974896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.526987806Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527000895Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:09:10.527013695Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527033085Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"":"downstairs","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527046135Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527057965Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"0","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527074005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"1","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527087565Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"client":"2","":"downstairs","session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527100595Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527112695Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"026af693-7750-42f0-991c-bc59f87c4f26 is now active with session: 2ff8c933-8da9-467d-b285-5913d079e01b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527125325Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527137254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912,"session_id":"2ff8c933-8da9-467d-b285-5913d079e01b"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527149394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} {"msg":"Activated sub_volume 026af693-7750-42f0-991c-bc59f87c4f26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:10.527160394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 469 len 36864 data: 13 14 12 10 8 10 9 10 9 02/30 Read block 337 len 28672 03/30 Read block 578 len 40960 04/30 Flush 05/30 Write block 276 len 28672 data: 11 9 12 14 15 16 15 06/30 Write block 367 len 36864 data: 10 10 10 11 9 9 7 11 12 07/30 Read block 244 len 28672 08/30 Write block 482 len 8192 data: 19 20 09/30 Read block 250 len 20480 10/30 Read block 32 len 16384 11/30 Write block 247 len 20480 data: 12 17 15 16 13 12/30 Read block 219 len 32768 13/30 Flush 14/30 Write block 515 len 4096 data: 13 15/30 Write block 103 len 16384 data: 8 9 10 9 16/30 Read block 309 len 24576 17/30 Read block 323 len 8192 18/30 Read block 136 len 20480 19/30 Write block 143 len 40960 data: 13 11 12 12 13 16 17 17 17 20 20/30 Write block 210 len 16384 data: 18 18 17 14 21/30 Read block 82 len 28672 22/30 Write block 128 len 36864 data: 16 18 20 25 21 19 18 15 17 23/30 Write block 31 len 24576 data: 15 15 16 19 20 24 24/30 Write block 110 len 16384 data: 11 9 11 11 25/30 Read block 222 len 20480 26/30 Read block 291 len 36864 27/30 Write block 41 len 16384 data: 20 16 15 13 28/30 Write block 551 len 16384 data: 10 14 15 14 29/30 Write block 186 len 12288 data: 16 15 11 30/30 Read block 52 len 28672 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 1 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 3 Sent Sent Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 15 15 17 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:10.547395503Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1912} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 301 301 301 F F F 1 020-039 174 174 174 291 291 291 T T T 2 040-059 174 174 174 291 291 291 T T T 3 060-079 176 176 176 293 293 293 F F F 4 080-099 179 179 179 301 301 301 F F F 5 100-119 179 179 179 301 301 301 T T T 6 120-139 176 176 176 294 294 294 T T T 7 140-159 178 178 178 300 300 300 T T T 8 160-179 178 178 178 297 297 297 F F F 9 180-199 174 174 174 291 291 291 T T T 10 200-219 179 179 179 301 301 301 T T T 11 220-239 179 179 179 301 301 301 F F F 12 240-259 180 180 180 303 303 303 F F F 13 260-279 180 180 180 303 303 303 F F F 14 280-299 180 180 180 303 303 303 F F F 15 300-319 179 179 179 301 301 301 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 180 180 180 303 303 303 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 178 178 178 299 299 299 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 180 180 180 302 302 302 F F F 24 480-499 180 180 180 303 303 303 F F F 25 500-519 179 179 179 301 301 301 T T T 26 520-539 174 174 174 291 291 291 F F F 27 540-559 179 179 179 301 301 301 T T T 28 560-579 179 179 179 301 301 301 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 180, Max flush: 303 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 301 301 301 F F F 1 020-039 181 181 181 304 304 304 F F F 2 040-059 181 181 181 304 304 304 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 179 179 179 301 301 301 F F F 5 100-119 181 181 181 304 304 304 F F F 6 120-139 181 181 181 304 304 304 F F F 7 140-159 181 181 181 304 304 304 F F F 8 160-179 178 178 178 297 297 297 F F F 9 180-199 181 181 181 304 304 304 F F F 10 200-219 181 181 181 304 304 304 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 180 180 180 303 303 303 F F F 13 260-279 180 180 180 303 303 303 F F F 14 280-299 180 180 180 303 303 303 F F F 15 300-319 179 179 179 301 301 301 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 180 180 180 303 303 303 F F F 19 380-399 163 163 163 268 268 268 F F F 20 400-419 172 172 172 288 288 288 F F F 21 420-439 178 178 178 299 299 299 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 180 180 180 302 302 302 F F F 24 480-499 180 180 180 303 303 303 F F F 25 500-519 181 181 181 304 304 304 F F F 26 520-539 174 174 174 291 291 291 F F F 27 540-559 181 181 181 304 304 304 F F F 28 560-579 179 179 179 301 301 301 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 181, Max flush: 304 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.810140732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:14.854848097Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.855235104Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.886550649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.887095565Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.887127935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.887141115Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.887155105Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} {"msg":"Crucible 32690ebf-dff9-4457-a043-42c3c8e015b5 has session id: 8e2eeffa-7a7f-4300-b5bf-2dcda799853f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.88775607Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Upstairs opts: Upstairs UUID: 32690ebf-dff9-4457-a043-42c3c8e015b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.88779365Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Crucible stats registered with UUID: 32690ebf-dff9-4457-a043-42c3c8e015b5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.88781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:14.88782649Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888244667Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888278986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} {"msg":"32690ebf-dff9-4457-a043-42c3c8e015b5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888379266Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888406375Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"0","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888433945Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"0","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888447625Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"1","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888462295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"1","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888475935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"2","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888490095Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"2","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888502825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888536554Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888565234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42316)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888592734Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42172)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888620714Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46502)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.888845492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.889330848Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"1","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.889359578Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"2","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.889554777Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"0","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.889981483Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"2","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890022453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"1","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890167642Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"0","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890558259Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890668658Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[0]R flush_numbers[0..12]: [301, 304, 304, 293, 301, 304, 304, 304, 297, 304, 304, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890761398Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[0]R generation[0..12]: [179, 181, 181, 176, 179, 181, 181, 181, 178, 181, 181, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890782317Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890797128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[1]R flush_numbers[0..12]: [301, 304, 304, 293, 301, 304, 304, 304, 297, 304, 304, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890810937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[1]R generation[0..12]: [179, 181, 181, 176, 179, 181, 181, 181, 178, 181, 181, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890825077Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890838937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[2]R flush_numbers[0..12]: [301, 304, 304, 293, 301, 304, 304, 304, 297, 304, 304, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890856057Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[2]R generation[0..12]: [179, 181, 181, 176, 179, 181, 181, 181, 178, 181, 181, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890869827Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890883407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890896087Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890909027Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890921576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890934156Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"":"downstairs","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890946896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890958696Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,Wait for a query_work_queue command to finish before sending IO "client":"0","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890978446Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"1","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.890991576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"client":"2","":"downstairs","session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.891004386Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.891016596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"32690ebf-dff9-4457-a043-42c3c8e015b5 is now active with session: 8e2eeffa-7a7f-4300-b5bf-2dcda799853f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.891029096Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.891040836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920,"session_id":"8e2eeffa-7a7f-4300-b5bf-2dcda799853f"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.891055055Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} {"msg":"Activated sub_volume 32690ebf-dff9-4457-a043-42c3c8e015b5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:14.891066155Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 519 len 28672 data: 20 20 23 22 14 13 11 02/30 Write block 52 len 32768 data: 10 11 11 13 13 12 12 13 03/30 Write block 134 len 12288 data: 19 16 18 04/30 Write block 115 len 36864 data: 10 10 10 11 14 14 13 14 17 05/30 Read block 82 len 36864 06/30 Read block 3 len 28672 07/30 Read block 217 len 32768 08/30 Read block 358 len 32768 09/30 Flush 10/30 Write block 166 len 28672 data: 17 17 13 17 17 14 15 11/30 Write block 107 len 28672 data: 8 10 13 12 10 12 12 12/30 Write block 261 len 40960 data: 10 11 14 11 16 15 15 15 14 15 13/30 Read block 383 len 32768 14/30 Read block 554 len 36864 15/30 Read block 390 len 4096 16/30 Flush 17/30 Write block 387 len 16384 data: 16 16 20 18 18/30 Read block 286 len 8192 19/30 Read block 51 len 16384 20/30 Write block 412 len 40960 data: 13 16 11 10 10 10 12 15 16 17 21/30 Read block 490 len 12288 22/30 Write block 310 len 12288 data: 10 9 10 23/30 Read block 355 len 16384 24/30 Write block 197 len 28672 data: 10 8 8 10 10 9 9 25/30 Read block 273 len 36864 26/30 Read block 320 len 16384 27/30 Read block 440 len 12288 28/30 Read block 321 len 16384 29/30 Read block 309 len 32768 30/30 Read block 183 len 20480 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:14.912646394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1920} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 301 301 301 F F F 1 020-039 181 181 181 304 304 304 F F F 2 040-059 182 182 182 305 305 305 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 179 179 179 301 301 301 F F F 5 100-119 182 182 182 306 306 306 F F F 6 120-139 182 182 182 305 305 305 F F F 7 140-159 181 181 181 304 304 304 F F F 8 160-179 182 182 182 306 306 306 F F F 9 180-199 181 181 181 304 304 304 T T T 10 200-219 181 181 181 304 304 304 T T T 11 220-239 179 179 179 301 301 301 F F F 12 240-259 180 180 180 303 303 303 F F F 13 260-279 182 182 182 306 306 306 F F F 14 280-299 180 180 180 303 303 303 F F F 15 300-319 179 179 179 301 301 301 T T T 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 180 180 180 303 303 303 F F F 19 380-399 163 163 163 268 268 268 T T T 20 400-419 172 172 172 288 288 288 T T T 21 420-439 178 178 178 299 299 299 T T T 22 440-459 178 178 178 297 297 297 F F F 23 460-479 180 180 180 302 302 302 F F F 24 480-499 180 180 180 303 303 303 F F F 25 500-519 182 182 182 305 305 305 F F F 26 520-539 182 182 182 305 305 305 F F F 27 540-559 181 181 181 304 304 304 F F F 28 560-579 179 179 179 301 301 301 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 182, Max flush: 306 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 301 301 301 F F F 1 020-039 181 181 181 304 304 304 F F F 2 040-059 182 182 182 305 305 305 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 179 179 179 301 301 301 F F F 5 100-119 182 182 182 306 306 306 F F F 6 120-139 182 182 182 305 305 305 F F F 7 140-159 181 181 181 304 304 304 F F F 8 160-179 182 182 182 306 306 306 F F F 9 180-199 183 183 183 307 307 307 F F F 10 200-219 183 183 183 307 307 307 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 180 180 180 303 303 303 F F F 13 260-279 182 182 182 306 306 306 F F F 14 280-299 180 180 180 303 303 303 F F F 15 300-319 183 183 183 307 307 307 F F F 16 320-339 159 159 159 264 264 264 F F F 17 340-359 174 174 174 291 291 291 F F F 18 360-379 180 180 180 303 303 303 F F F 19 380-399 183 183 183 307 307 307 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 183 183 183 307 307 307 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 180 180 180 302 302 302 F F F 24 480-499 180 180 180 303 303 303 F F F 25 500-519 182 182 182 305 305 305 F F F 26 520-539 182 182 182 305 305 305 F F F 27 540-559 181 181 181 304 304 304 F F F 28 560-579 179 179 179 301 301 301 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 183, Max flush: 307 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.18390234Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:19.226415041Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.226772748Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.257649367Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.258175973Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.258216913Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.258233773Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.258252273Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} {"msg":"Crucible 7ee12983-38da-40bf-9559-b7e2ae13eaaa has session id: 714ffd29-cc51-436b-a248-af3c820215d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.258767939Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Upstairs opts: Upstairs UUID: 7ee12983-38da-40bf-9559-b7e2ae13eaaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.258813879Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Crucible stats registered with UUID: 7ee12983-38da-40bf-9559-b7e2ae13eaaa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.258832219Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:19.258846758Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259348285Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259372935Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} {"msg":"7ee12983-38da-40bf-9559-b7e2ae13eaaa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259450574Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259468394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"0","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259495743Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"0","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259510094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"1","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259529323Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"1","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259543143Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"2","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259557043Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"2","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259575933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259599253Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259625513Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43578)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259669152Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44728)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.259698912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35511)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.25998332Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.260444507Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"2","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.260534426Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"1","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.260578675Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"0","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261074472Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"2","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.26125327Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"1","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261370469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"0","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261720037Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261808926Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[0]R flush_numbers[0..12]: [301, 304, 305, 293, 301, 306, 305, 304, 306, 307, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261929205Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[0]R generation[0..12]: [179, 181, 182, 176, 179, 182, 182, 181, 182, 183, 183, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261949825Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261964835Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[1]R flush_numbers[0..12]: [301, 304, 305, 293, 301, 306, 305, 304, 306, 307, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261979005Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[1]R generation[0..12]: [179, 181, 182, 176, 179, 182, 182, 181, 182, 183, 183, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.261993235Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262008585Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[2]R flush_numbers[0..12]: [301, 304, 305, 293, 301, 306, 305, 304, 306, 307, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262022415Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[2]R generation[0..12]: [179, 181, 182, 176, 179, 182, 182, 181, 182, 183, 183, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262035964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262049724Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262062694Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262075684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 308","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262099004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262112704Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"":"downstairs","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262125744Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262137744Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"0","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262150904Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"1","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262163723Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"client":"2","":"downstairs","session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262176843Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262188933Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"7ee12983-38da-40bf-9559-b7e2ae13eaaa is now active with session: 714ffd29-cc51-436b-a248-af3c820215d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262201403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262213273Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928,"session_id":"714ffd29-cc51-436b-a248-af3c820215d0"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262225343Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} {"msg":"Activated sub_volume 7ee12983-38da-40bf-9559-b7e2ae13eaaa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:19.262236433Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 24576 data: 19 23 21 22 17 11 02/30 Read block 107 len 24576 03/30 Read block 19 len 16384 04/30 Read block 93 len 36864 05/30 Write block 465 len 28672 data: 15 13 13 13 14 15 13 06/30 Flush 07/30 Write block 84 len 12288 data: 12 11 12 08/30 Write block 561 len 32768 data: 16 13 12 12 13 11 9 11 09/30 Read block 371 len 28672 10/30 Read block 229 len 32768 11/30 Read block 244 len 32768 12/30 Read block 458 len 16384 13/30 Write block 334 len 28672 data: 11 12 14 15 16 14 12 14/30 Read block 579 len 24576 15/30 Flush 16/30 Write block 89 len 28672 data: 14 16 14 16 17 19 15 17/30 Write block 330 len 8192 data: 15 11 18/30 Read block 134 len 40960 19/30 Write block 566 len 28672 data: 12 10 12 11 13 12 12 20/30 Read block 393 len 12288 21/30 Read block 238 len 4096 22/30 Read block 316 len 8192 23/30 Write block 207 len 8192 data: 12 15 24/30 Flush 25/30 Write block 113 len 40960 data: 13 12 11 11 11 12 15 15 14 15 26/30 Read block 65 len 20480 27/30 Write block 246 len 32768 data: 11 13 18 16 17 14 13 12 28/30 Read block 287 len 20480 29/30 Read block 304 len 36864 30/30 Write block 343 len 8192 data: 13 13 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:19.283719662Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 301 301 301 F F F 1 020-039 181 181 181 304 304 304 F F F 2 040-059 182 182 182 305 305 305 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 184 184 184 310 310 310 F F F 5 100-119 182 182 182 306 306 306 T T T 6 120-139 182 182 182 305 305 305 T T T 7 140-159 181 181 181 304 304 304 F F F 8 160-179 182 182 182 306 306 306 F F F 9 180-199 183 183 183 307 307 307 F F F 10 200-219 184 184 184 310 310 310 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 180 180 180 303 303 303 T T T 13 260-279 182 182 182 306 306 306 F F F 14 280-299 180 180 180 303 303 303 F F F 15 300-319 183 183 183 307 307 307 F F F 16 320-339 184 184 184 310 310 310 F F F 17 340-359 184 184 184 309 309 309 T T T 18 360-379 180 180 180 303 303 303 F F F 19 380-399 183 183 183 307 307 307 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 183 183 183 307 307 307 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 184 184 184 308 308 308 F F F 24 480-499 180 180 180 303 303 303 F F F 25 500-519 182 182 182 305 305 305 F F F 26 520-539 182 182 182 305 305 305 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 184 184 184 310 310 310 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 184, Max flush: 310 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 301 301 301 F F F 1 020-039 181 181 181 304 304 304 F F F 2 040-059 182 182 182 305 305 305 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 184 184 184 310 310 310 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 185 185 185 311 311 311 F F F 7 140-159 181 181 181 304 304 304 F F F 8 160-179 182 182 182 306 306 306 F F F 9 180-199 183 183 183 307 307 307 F F F 10 200-219 184 184 184 310 310 310 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 185 185 185 311 311 311 F F F 13 260-279 182 182 182 306 306 306 F F F 14 280-299 180 180 180 303 303 303 F F F 15 300-319 183 183 183 307 307 307 F F F 16 320-339 184 184 184 310 310 310 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 180 180 180 303 303 303 F F F 19 380-399 183 183 183 307 307 307 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 183 183 183 307 307 307 F F F 22 440-459 178 178 178 297 297 297 F F F 23 460-479 184 184 184 308 308 308 F F F 24 480-499 180 180 180 303 303 303 F F F 25 500-519 182 182 182 305 305 305 F F F 26 520-539 182 182 182 305 305 305 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 184 184 184 310 310 310 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 185, Max flush: 311 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.546864379Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:23.588678386Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.589033393Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.620060791Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.620596557Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.620632317Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.620646826Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.620661926Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} {"msg":"Crucible d6ca061f-eeb9-484e-b7db-98f6806b49da has session id: 0525b9cb-0c20-4f97-bae5-17d233e5326e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621271702Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Upstairs opts: Upstairs UUID: d6ca061f-eeb9-484e-b7db-98f6806b49da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621300981Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Crucible stats registered with UUID: d6ca061f-eeb9-484e-b7db-98f6806b49da","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621318901Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:23.621331791Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621732078Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621756178Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} {"msg":"d6ca061f-eeb9-484e-b7db-98f6806b49da active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621824678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621842517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"0","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621868987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"0","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621883737Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"1","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621899677Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"1","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621922947Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"2","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621948407Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"2","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.621973587Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.622018166Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.622052016Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"ds_connection connected from Ok(127.0.0.1:65184)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.622079866Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38915)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.622111486Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63139)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.622136165Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62280946Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"1","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62284609Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"2","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62286912Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"0","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.623492945Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"1","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.623532405Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"0","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.623560765Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"2","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62412452Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62415082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[0]R flush_numbers[0..12]: [301, 304, 305, 293, 310, 311, 311, 304, 306, 307, 310, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62419238Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[0]R generation[0..12]: [179, 181, 182, 176, 184, 185, 185, 181, 182, 183, 184, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62421105Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62422582Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[1]R flush_numbers[0..12]: [301, 304, 305, 293, 310, 311, 311, 304, 306, 307, 310, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624239539Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[1]R generation[0..12]: [179, 181, 182, 176, 184, 185, 185, 181, 182, 183, 184, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.62425779Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624272559Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[2]R flush_numbers[0..12]: [301, 304, 305, 293, 310, 311, 311, 304, 306, 307, 310, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624286749Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[2]R generation[0..12]: [179, 181, 182, 176, 184, 185, 185, 181, 182, 183, 184, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624300699Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624314599Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624327249Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624340209Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624353059Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624365719Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"":"downstairs","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624378978Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624390958Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"0","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624404258Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"1","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-18T05:09:23.624423878Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"client":"2","":"downstairs","session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624436968Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624448918Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"d6ca061f-eeb9-484e-b7db-98f6806b49da is now active with session: 0525b9cb-0c20-4f97-bae5-17d233e5326e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624464668Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624477778Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936,"session_id":"0525b9cb-0c20-4f97-bae5-17d233e5326e"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624490068Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} {"msg":"Activated sub_volume d6ca061f-eeb9-484e-b7db-98f6806b49da","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:23.624501197Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 13 len 32768 data: 18 19 18 17 17 16 16 16 02/30 Read block 70 len 4096 03/30 Read block 221 len 8192 04/30 Read block 464 len 24576 05/30 Write block 478 len 4096 data: 11 06/30 Read block 40 len 24576 07/30 Read block 301 len 32768 08/30 Read block 172 len 8192 09/30 Write block 513 len 4096 data: 14 10/30 Read block 20 len 32768 11/30 Read block 330 len 16384 12/30 Read block 340 len 8192 13/30 Read block 470 len 16384 14/30 Flush 15/30 Write block 166 len 20480 data: 18 18 14 18 18 16/30 Read block 381 len 12288 17/30 Read block 489 len 16384 18/30 Write block 456 len 20480 data: 12 12 8 8 8 19/30 Write block 360 len 24576 data: 12 11 9 9 11 8 20/30 Write block 306 len 32768 data: 11 11 10 10 11 10 11 9 21/30 Write block 497 len 40960 data: 14 15 16 15 15 12 13 13 12 13 22/30 Read block 324 len 16384 23/30 Read block 206 len 4096 24/30 Read block 80 len 4096 25/30 Write block 191 len 12288 data: 11 10 9 26/30 Read block 182 len 20480 27/30 Write block 193 len 28672 data: 10 9 11 11 11 9 9 28/30 Write block 274 len 4096 data: 12 29/30 Read block 221 len 40960 30/30 Write block 211 len 4096 data: 19 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:23.643734844Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1936} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 312 312 312 F F F 1 020-039 186 186 186 312 312 312 F F F 2 040-059 182 182 182 305 305 305 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 184 184 184 310 310 310 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 185 185 185 311 311 311 F F F 7 140-159 181 181 181 304 304 304 F F F 8 160-179 182 182 182 306 306 306 T T T 9 180-199 183 183 183 307 307 307 T T T 10 200-219 184 184 184 310 310 310 T T T 11 220-239 179 179 179 301 301 301 F F F 12 240-259 185 185 185 311 311 311 F F F 13 260-279 182 182 182 306 306 306 T T T 14 280-299 180 180 180 303 303 303 F F F 15 300-319 183 183 183 307 307 307 T T T 16 320-339 184 184 184 310 310 310 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 180 180 180 303 303 303 T T T 19 380-399 183 183 183 307 307 307 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 183 183 183 307 307 307 F F F 22 440-459 178 178 178 297 297 297 T T T 23 460-479 186 186 186 312 312 312 T T T 24 480-499 180 180 180 303 303 303 T T T 25 500-519 186 186 186 312 312 312 T T T 26 520-539 182 182 182 305 305 305 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 184 184 184 310 310 310 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 186, Max flush: 312 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 312 312 312 F F F 1 020-039 186 186 186 312 312 312 F F F 2 040-059 182 182 182 305 305 305 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 184 184 184 310 310 310 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 185 185 185 311 311 311 F F F 7 140-159 181 181 181 304 304 304 F F F 8 160-179 187 187 187 313 313 313 F F F 9 180-199 187 187 187 313 313 313 F F F 10 200-219 187 187 187 313 313 313 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 185 185 185 311 311 311 F F F 13 260-279 187 187 187 313 313 313 F F F 14 280-299 180 180 180 303 303 303 F F F 15 300-319 187 187 187 313 313 313 F F F 16 320-339 184 184 184 310 310 310 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 183 183 183 307 307 307 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 183 183 183 307 307 307 F F F 22 440-459 187 187 187 313 313 313 F F F 23 460-479 187 187 187 313 313 313 F F F 24 480-499 187 187 187 313 313 313 F F F 25 500-519 187 187 187 313 313 313 F F F 26 520-539 182 182 182 305 305 305 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 184 184 184 310 310 310 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 187, Max flush: 313 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.937766089Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.981784809Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.982336815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.982394555Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.982417715Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.982441834Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} {"msg":"Crucible 6ca5cbbc-4630-4df9-a611-64f0255c87f1 has session id: e2ac21c2-46af-4cd7-9be4-a5438c87e02d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.982899651Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Upstairs opts: Upstairs UUID: 6ca5cbbc-4630-4df9-a611-64f0255c87f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.982941411Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Crucible stats registered with UUID: 6ca5cbbc-4630-4df9-a611-64f0255c87f1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.98296686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:27.98300469Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983497926Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983522826Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} {"msg":"6ca5cbbc-4630-4df9-a611-64f0255c87f1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983594116Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983611815Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"0","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983638435Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"0","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983652335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"1","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983670945Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"1","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983694745Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983721885Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983745494Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983768484Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983795594Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:27.983826884Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:27.983859144Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:27.983887753Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47665)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983911163Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55769)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.983937613Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.984030942Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.984477049Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"0","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.984506389Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"1","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.985131254Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"0","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.985170564Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"1","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.98571385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:27.985827749Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.985974627Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45542)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.986239185Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.987388706Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.987912512Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[0]R flush_numbers[0..12]: [312, 312, 305, 293, 310, 311, 311, 304, 313, 313, 313, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988518328Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[0]R generation[0..12]: [186, 186, 182, 176, 184, 185, 185, 181, 187, 187, 187, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988543288Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988561637Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[1]R flush_numbers[0..12]: [312, 312, 305, 293, 310, 311, 311, 304, 313, 313, 313, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988579017Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[1]R generation[0..12]: [186, 186, 182, 176, 184, 185, 185, 181, 187, 187, 187, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988595867Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988613017Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[2]R flush_numbers[0..12]: [312, 312, 305, 293, 310, 311, 311, 304, 313, 313, 313, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988635777Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[2]R generation[0..12]: [186, 186, 182, 176, 184, 185, 185, 181, 187, 187, 187, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988653367Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988670367Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988686207Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988704006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988720106Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988735976Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"":"downstairs","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988752206Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988766956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"0","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988781896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"1","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988794566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"client":"2","":"downstairs","session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988807446Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988819535Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {Wait for a query_work_queue command to finish before sending IO "msg":"6ca5cbbc-4630-4df9-a611-64f0255c87f1 is now active with session: e2ac21c2-46af-4cd7-9be4-a5438c87e02d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988839615Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988851495Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944,"session_id":"e2ac21c2-46af-4cd7-9be4-a5438c87e02d"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988865795Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} {"msg":"Activated sub_volume 6ca5cbbc-4630-4df9-a611-64f0255c87f1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:37.988877035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 32768 data: 10 14 14 17 17 21 21 24 02/30 Write block 41 len 36864 data: 21 17 16 14 11 9 11 12 12 03/30 Read block 590 len 8192 04/30 Flush 05/30 Write block 244 len 40960 data: 13 11 12 14 19 17 18 15 14 13 06/30 Flush 07/30 Write block 133 len 16384 data: 20 20 17 19 08/30 Read block 305 len 12288 09/30 Read block 429 len 12288 10/30 Read block 275 len 40960 11/30 Read block 508 len 28672 12/30 Read block 153 len 8192 13/30 Write block 527 len 28672 data: 12 11 12 10 10 9 10 14/30 Write block 20 len 36864 data: 17 19 18 21 18 15 17 14 12 15/30 Read block 534 len 40960 16/30 Write block 11 len 8192 data: 22 19 17/30 Read block 142 len 20480 18/30 Read block 521 len 24576 19/30 Read block 103 len 12288 20/30 Write block 242 len 28672 data: 14 13 14 12 13 15 20 21/30 Read block 69 len 20480 22/30 Read block 91 len 20480 23/30 Read block 428 len 20480 24/30 Write block 422 len 24576 data: 16 16 14 14 13 13 25/30 Write block 287 len 32768 data: 18 19 18 20 22 23 23 21 26/30 Read block 350 len 28672 27/30 Read block 494 len 40960 28/30 Write block 295 len 4096 data: 19 29/30 Read block 273 len 40960 30/30 Read block 337 len 20480 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 24 23 23 70 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:38.011164458Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1944} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 312 312 312 T T T 1 020-039 186 186 186 312 312 312 T T T 2 040-059 188 188 188 314 314 314 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 184 184 184 310 310 310 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 185 185 185 311 311 311 T T T 7 140-159 181 181 181 304 304 304 F F F 8 160-179 187 187 187 313 313 313 F F F 9 180-199 187 187 187 313 313 313 F F F 10 200-219 187 187 187 313 313 313 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 188 188 188 315 315 315 T T T 13 260-279 187 187 187 313 313 313 F F F 14 280-299 180 180 180 303 303 303 T T T 15 300-319 187 187 187 313 313 313 F F F 16 320-339 184 184 184 310 310 310 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 183 183 183 307 307 307 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 183 183 183 307 307 307 T T T 22 440-459 187 187 187 313 313 313 F F F 23 460-479 187 187 187 313 313 313 F F F 24 480-499 187 187 187 313 313 313 F F F 25 500-519 188 188 188 314 314 314 F F F 26 520-539 188 188 188 314 314 314 T T T 27 540-559 184 184 184 308 308 308 F F F 28 560-579 184 184 184 310 310 310 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 188, Max flush: 315 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 189 189 189 316 316 316 F F F 2 040-059 188 188 188 314 314 314 F F F 3 060-079 176 176 176 293 293 293 F F F 4 080-099 184 184 184 310 310 310 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 189 189 189 316 316 316 F F F 7 140-159 181 181 181 304 304 304 F F F 8 160-179 187 187 187 313 313 313 F F F 9 180-199 187 187 187 313 313 313 F F F 10 200-219 187 187 187 313 313 313 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 187 187 187 313 313 313 F F F 14 280-299 189 189 189 316 316 316 F F F 15 300-319 187 187 187 313 313 313 F F F 16 320-339 184 184 184 310 310 310 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 183 183 183 307 307 307 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 189 189 189 316 316 316 F F F 22 440-459 187 187 187 313 313 313 F F F 23 460-479 187 187 187 313 313 313 F F F 24 480-499 187 187 187 313 313 313 F F F 25 500-519 188 188 188 314 314 314 F F F 26 520-539 189 189 189 316 316 316 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 184 184 184 310 310 310 F F F 29 580-599 172 172 172 284 284 284 F F F Max gen: 189, Max flush: 316 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.26429987Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:42.312620488Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.312980035Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.343893894Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.344528079Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.344561149Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.344574949Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.344589058Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} {"msg":"Crucible d73cb44b-aac0-49a7-b528-5938d67d5d8a has session id: fb5f5014-974a-44ee-b7f9-e163a68b8062","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345143284Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Upstairs opts: Upstairs UUID: d73cb44b-aac0-49a7-b528-5938d67d5d8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345209274Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Crucible stats registered with UUID: d73cb44b-aac0-49a7-b528-5938d67d5d8a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345235184Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:42.345248624Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345619801Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34564453Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} {"msg":"d73cb44b-aac0-49a7-b528-5938d67d5d8a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34571776Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34573593Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"0","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34577458Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"0","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345801449Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"1","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345829709Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"1","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345857129Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"2","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345884859Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"2","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345909588Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345936468Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345963848Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"ds_connection connected from Ok(127.0.0.1:41266)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.345991368Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"ds_connection connected from Ok(127.0.0.1:59017)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.346017898Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"ds_connection connected from Ok(127.0.0.1:56904)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.346046478Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.346644883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"1","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.346682703Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"2","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.346713173Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"0","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.347354858Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"1","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.347396997Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"2","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.347473777Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"0","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348034603Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348070832Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[0]R flush_numbers[0..12]: [316, 316, 314, 293, 310, 311, 316, 304, 313, 313, 313, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348129162Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[0]R generation[0..12]: [189, 189, 188, 176, 184, 185, 189, 181, 187, 187, 187, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348157722Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348173131Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[1]R flush_numbers[0..12]: [316, 316, 314, 293, 310, 311, 316, 304, 313, 313, 313, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348187231Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[1]R generation[0..12]: [189, 189, 188, 176, 184, 185, 189, 181, 187, 187, 187, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348200751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348214621Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[2]R flush_numbers[0..12]: [316, 316, 314, 293, 310, 311, 316, 304, 313, 313, 313, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348229091Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[2]R generation[0..12]: [189, 189, 188, 176, 184, 185, 189, 181, 187, 187, 187, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348242661Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348256491Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348272951Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348286451Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34829931Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34831247Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"":"downstairs","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34832546Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34833747Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX"Wait for a query_work_queue command to finish before sending IO ,"pid":1952,"client":"0","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34835812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"1","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34837134Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"client":"2","":"downstairs","session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34838446Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34839682Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"d73cb44b-aac0-49a7-b528-5938d67d5d8a is now active with session: fb5f5014-974a-44ee-b7f9-e163a68b8062","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34840956Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.34842137Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952,"session_id":"fb5f5014-974a-44ee-b7f9-e163a68b8062"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348433989Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} {"msg":"Activated sub_volume d73cb44b-aac0-49a7-b528-5938d67d5d8a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:42.348445189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 32768 data: 16 13 14 14 13 14 14 12 02/30 Write block 140 len 40960 data: 15 13 13 14 12 13 13 14 17 18 03/30 Read block 392 len 32768 04/30 Write block 579 len 40960 data: 17 17 14 15 14 13 14 15 16 15 05/30 Write block 120 len 20480 data: 16 15 16 18 20 06/30 Write block 444 len 12288 data: 12 10 12 07/30 Read block 31 len 36864 08/30 Flush 09/30 Write block 500 len 4096 data: 16 10/30 Read block 340 len 4096 11/30 Write block 435 len 16384 data: 16 14 13 15 12/30 Read block 9 len 16384 13/30 Write block 204 len 24576 data: 9 10 12 13 16 15 14/30 Write block 73 len 12288 data: 16 14 19 15/30 Read block 558 len 4096 16/30 Write block 203 len 8192 data: 10 10 17/30 Write block 496 len 40960 data: 13 15 16 17 17 17 14 15 15 14 18/30 Write block 190 len 12288 data: 12 12 11 19/30 Write block 524 len 4096 data: 14 20/30 Read block 356 len 36864 21/30 Read block 123 len 40960 22/30 Read block 576 len 8192 23/30 Read block 175 len 28672 24/30 Write block 497 len 4096 data: 16 25/30 Read block 527 len 32768 26/30 Read block 523 len 20480 27/30 Read block 221 len 16384 28/30 Write block 128 len 12288 data: 17 19 21 29/30 Write block 383 len 32768 data: 13 16 20 21 17 17 21 19 30/30 Write block 274 len 28672 data: 13 16 12 10 13 15 16 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 1 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:42.366815702Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1952} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 189 189 189 316 316 316 F F F 2 040-059 188 188 188 314 314 314 F F F 3 060-079 176 176 176 293 293 293 T T T 4 080-099 184 184 184 310 310 310 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 190 190 190 317 317 317 T T T 7 140-159 190 190 190 317 317 317 F F F 8 160-179 187 187 187 313 313 313 F F F 9 180-199 187 187 187 313 313 313 T T T 10 200-219 187 187 187 313 313 313 T T T 11 220-239 179 179 179 301 301 301 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 187 187 187 313 313 313 T T T 14 280-299 189 189 189 316 316 316 T T T 15 300-319 187 187 187 313 313 313 F F F 16 320-339 184 184 184 310 310 310 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 183 183 183 307 307 307 T T T 20 400-419 183 183 183 307 307 307 F F F 21 420-439 189 189 189 316 316 316 T T T 22 440-459 190 190 190 317 317 317 F F F 23 460-479 187 187 187 313 313 313 F F F 24 480-499 187 187 187 313 313 313 T T T 25 500-519 190 190 190 317 317 317 T T T 26 520-539 189 189 189 316 316 316 T T T 27 540-559 184 184 184 308 308 308 F F F 28 560-579 190 190 190 317 317 317 F F F 29 580-599 190 190 190 317 317 317 F F F Max gen: 190, Max flush: 317 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 316 316 316 F F F 1 020-039 189 189 189 316 316 316 F F F 2 040-059 188 188 188 314 314 314 F F F 3 060-079 191 191 191 318 318 318 F F F 4 080-099 184 184 184 310 310 310 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 191 191 191 318 318 318 F F F 7 140-159 190 190 190 317 317 317 F F F 8 160-179 187 187 187 313 313 313 F F F 9 180-199 191 191 191 318 318 318 F F F 10 200-219 191 191 191 318 318 318 F F F 11 220-239 179 179 179 301 301 301 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 191 191 191 318 318 318 F F F 14 280-299 191 191 191 318 318 318 F F F 15 300-319 187 187 187 313 313 313 F F F 16 320-339 184 184 184 310 310 310 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 191 191 191 318 318 318 F F F 22 440-459 190 190 190 317 317 317 F F F 23 460-479 187 187 187 313 313 313 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 191 191 191 318 318 318 F F F 26 520-539 191 191 191 318 318 318 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 190 190 190 317 317 317 F F F 29 580-599 190 190 190 317 317 317 F F F Max gen: 191, Max flush: 318 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.662329505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.706408275Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.707054401Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70709641Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70711435Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70713298Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} {"msg":"Crucible fd5841f8-9872-4f66-ae17-588ebf88c38f has session id: 0866f244-8bb3-410c-877f-c252e8643c6b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.707643626Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Upstairs opts: Upstairs UUID: fd5841f8-9872-4f66-ae17-588ebf88c38f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.707683366Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Crucible stats registered with UUID: fd5841f8-9872-4f66-ae17-588ebf88c38f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.707709576Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:46.707731335Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708240261Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708274851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} {"msg":"fd5841f8-9872-4f66-ae17-588ebf88c38f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708337121Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708363081Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"0","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70839705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"0","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70842311Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"1","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70845101Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"1","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70847648Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"2","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.70850403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"2","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708535519Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708562499Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708588729Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49667)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708616529Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:53609)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708642709Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65506)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.708667188Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.709334903Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"0","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.709363683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"1","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.709380533Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"2","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.709979119Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"1","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710103108Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"0","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710170677Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"2","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710495875Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710583454Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[0]R flush_numbers[0..12]: [316, 316, 314, 318, 310, 311, 318, 317, 313, 318, 318, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710704393Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[0]R generation[0..12]: [189, 189, 188, 191, 184, 185, 191, 190, 187, 191, 191, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710728083Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710742963Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[1]R flush_numbers[0..12]: [316, 316, 314, 318, 310, 311, 318, 317, 313, 318, 318, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710757493Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[1]R generation[0..12]: [189, 189, 188, 191, 184, 185, 191, 190, 187, 191, 191, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710771203Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710785082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[2]R flush_numbers[0..12]: [316, 316, 314, 318, 310, 311, 318, 317, 313, 318, 318, 301]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710799042Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[2]R generation[0..12]: [189, 189, 188, 191, 184, 185, 191, 190, 187, 191, 191, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710812812Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710826562Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710839312Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710852322Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710865032Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710877762Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"":"downstairs","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710890862Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710902692Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"0","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710917712Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"1","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710938721Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"client":"2","":"downstairs","session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710952061Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710964281Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"fd5841f8-9872-4f66-ae17-588ebf88c38f is now active with session: 0866f244-8bb3-410c-877f-c252e8643c6b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710976851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.710988781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960,"session_id":"0866f244-8bb3-410c-877f-c252e8643c6b"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.711000961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} {"msg":"Activated sub_volume fd5841f8-9872-4f66-ae17-588ebf88c38f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:46.711012091Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 291 len 32768 data: 23 24 24 22 20 18 17 15 02/30 Read block 261 len 32768 03/30 Read block 352 len 32768 04/30 Read block 132 len 4096 05/30 Read block 268 len 40960 06/30 Read block 352 len 40960 07/30 Write block 474 len 8192 data: 11 10 08/30 Read block 500 len 32768 09/30 Read block 169 len 40960 10/30 Flush 11/30 Write block 76 len 4096 data: 16 12/30 Read block 167 len 32768 13/30 Write block 207 len 16384 data: 14 17 16 19 14/30 Write block 38 len 20480 data: 25 26 21 22 18 15/30 Read block 283 len 12288 16/30 Write block 144 len 32768 data: 13 14 14 15 18 19 18 18 17/30 Read block 240 len 28672 18/30 Write block 224 len 4096 data: 15 19/30 Read block 506 len 32768 20/30 Write block 80 len 24576 data: 11 12 13 11 13 12 21/30 Read block 395 len 12288 22/30 Write block 317 len 20480 data: 11 11 11 14 17 23/30 Read block 210 len 32768 24/30 Read block 539 len 4096 25/30 Write block 15 len 40960 data: 19 18 18 17 17 18 20 19 22 19 26/30 Flush 27/30 Write block 317 len 8192 data: 12 12 28/30 Write block 198 len 28672 data: 10 10 11 11 10 11 11 29/30 Read block 187 len 20480 30/30 Write block 20 len 40960 data: 19 21 20 23 20 16 18 15 13 9 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:46.732369851Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1960} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 320 320 320 F F F 1 020-039 192 192 192 320 320 320 T T T 2 040-059 192 192 192 320 320 320 F F F 3 060-079 192 192 192 320 320 320 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 191 191 191 318 318 318 F F F 7 140-159 192 192 192 320 320 320 F F F 8 160-179 187 187 187 313 313 313 F F F 9 180-199 191 191 191 318 318 318 T T T 10 200-219 192 192 192 320 320 320 T T T 11 220-239 192 192 192 320 320 320 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 191 191 191 318 318 318 F F F 14 280-299 192 192 192 319 319 319 F F F 15 300-319 192 192 192 320 320 320 T T T 16 320-339 192 192 192 320 320 320 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 191 191 191 318 318 318 F F F 22 440-459 190 190 190 317 317 317 F F F 23 460-479 192 192 192 319 319 319 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 191 191 191 318 318 318 F F F 26 520-539 191 191 191 318 318 318 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 190 190 190 317 317 317 F F F 29 580-599 190 190 190 317 317 317 F F F Max gen: 192, Max flush: 320 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 320 320 320 F F F 1 020-039 193 193 193 321 321 321 F F F 2 040-059 192 192 192 320 320 320 F F F 3 060-079 192 192 192 320 320 320 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 185 185 185 311 311 311 F F F 6 120-139 191 191 191 318 318 318 F F F 7 140-159 192 192 192 320 320 320 F F F 8 160-179 187 187 187 313 313 313 F F F 9 180-199 193 193 193 321 321 321 F F F 10 200-219 193 193 193 321 321 321 F F F 11 220-239 192 192 192 320 320 320 F F F 12 240-259 189 189 189 316 316 316 F F F 13 260-279 191 191 191 318 318 318 F F F 14 280-299 192 192 192 319 319 319 F F F 15 300-319 193 193 193 321 321 321 F F F 16 320-339 192 192 192 320 320 320 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 191 191 191 318 318 318 F F F 22 440-459 190 190 190 317 317 317 F F F 23 460-479 192 192 192 319 319 319 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 191 191 191 318 318 318 F F F 26 520-539 191 191 191 318 318 318 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 190 190 190 317 317 317 F F F 29 580-599 190 190 190 317 317 317 F F F Max gen: 193, Max flush: 321 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:50.969514882Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.014639604Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.015308849Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.015364648Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.015384518Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.015402948Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} {"msg":"Crucible 65b78e0f-b844-476a-a1bc-3243b58d6bd6 has session id: 29a4d5a3-c3be-4c55-a619-b1a79e77baec","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.015956394Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Upstairs opts: Upstairs UUID: 65b78e0f-b844-476a-a1bc-3243b58d6bd6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016029043Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Crucible stats registered with UUID: 65b78e0f-b844-476a-a1bc-3243b58d6bd6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016063403Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:51.016081683Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016618189Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016653179Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} {"msg":"65b78e0f-b844-476a-a1bc-3243b58d6bd6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016753678Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016781998Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"0","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016828268Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"0","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016845128Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"1","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016859537Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"1","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016873007Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"2","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016886957Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"2","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016899947Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016913517Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.016927577Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"ds_connection connected from Ok(127.0.0.1:61412)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.017087836Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"ds_connection connected from Ok(127.0.0.1:61939)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.017127105Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"ds_connection connected from Ok(127.0.0.1:56240)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.017153105Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.01790826Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"2","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.017944959Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"0","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.017961489Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"1","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.018592484Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"2","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.018752863Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"1","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.018885062Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"0","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.01922565Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019376249Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[0]R flush_numbers[0..12]: [320, 321, 320, 320, 320, 311, 318, 320, 313, 321, 321, 320]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019501868Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[0]R generation[0..12]: [192, 193, 192, 192, 192, 185, 191, 192, 187, 193, 193, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019523248Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019538117Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[1]R flush_numbers[0..12]: [320, 321, 320, 320, 320, 311, 318, 320, 313, 321, 321, 320]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019552357Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[1]R generation[0..12]: [192, 193, 192, 192, 192, 185, 191, 192, 187, 193, 193, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019566077Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019579937Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[2]R flush_numbers[0..12]: [320, 321, 320, 320, 320, 311, 318, 320, 313, 321, 321, 320]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019593607Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[2]R generation[0..12]: [192, 193, 192, 192, 192, 185, 191, 192, 187, 193, 193, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019607277Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019620997Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019637896Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019652287Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019665246Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019678076Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"":"downstairs","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019698986Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019711236Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"0","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019724486Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"1","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019737686Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"client":"2","":"downstairs","session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019750786Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019763176Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"65b78e0f-b844-476a-a1bc-3243b58d6bd6 is now active with session: 29a4d5a3-c3be-4c55-a619-b1a79e77baec","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019776046Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019787955Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968,"session_id":"29a4d5a3-c3be-4c55-a619-b1a79e77baec"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019800275Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} {"msg":"Activated sub_volume 65b78e0f-b844-476a-a1bc-3243b58d6bd6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.019811465Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 75 len 16384 data: 20 17 15 17 02/30 Flush 03/30 Write block 22 len 32768 data: 21 24 21 17 19 16 14 10 04/30 Write block 305 len 28672 data: 13 12 12 11 11 12 11 05/30 Read block 428 len 8192 06/30 Write block 174 len 20480 data: 16 14 14 13 12 07/30 Read block 105 len 20480 08/30 Read block 421 len 24576 09/30 Write block 66 len 24576 data: 16 15 15 16 15 14 10/30 Write block 240 len 8192 data: 13 15 11/30 Flush 12/30 Write block 230 len 20480 data: 14 10 10 10 11 13/30 Write block 14 len 24576 data: 20 20 19 19 18 18 14/30 Read block 59 len 32768 15/30 Write block 284 len 16384 data: 17 19 16 19 16/30 Read block 533 len 32768 17/30 Flush 18/30 Write block 110 len 16384 data: 13 11 13 14 19/30 Write block 158 len 12288 data: 15 14 11 20/30 Read block 210 len 40960 21/30 Write block 513 len 28672 data: 15 11 15 15 18 18 22 22/30 Read block 149 len 24576 23/30 Read block 89 len 12288 24/30 Write block 18 len 16384 data: 19 19 20 22 25/30 Write block 123 len 12288 data: 19 21 18 26/30 Read block 426 len 24576 27/30 Read block 156 len 28672 28/30 Read block 454 len 32768 29/30 Write block 43 len 16384 data: 17 15 12 10 30/30 Write block 580 len 20480 data: 18 15 16 15 14 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:51.039494058Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1968} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 324 324 324 T T T 1 020-039 194 194 194 323 323 323 T T T 2 040-059 192 192 192 320 320 320 T T T 3 060-079 194 194 194 323 323 323 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 185 185 185 311 311 311 T T T 6 120-139 191 191 191 318 318 318 T T T 7 140-159 192 192 192 320 320 320 T T T 8 160-179 194 194 194 323 323 323 T T T 9 180-199 193 193 193 321 321 321 F F F 10 200-219 193 193 193 321 321 321 F F F 11 220-239 194 194 194 324 324 324 F F F 12 240-259 194 194 194 323 323 323 F F F 13 260-279 191 191 191 318 318 318 F F F 14 280-299 194 194 194 324 324 324 F F F 15 300-319 194 194 194 323 323 323 F F F 16 320-339 192 192 192 320 320 320 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 191 191 191 318 318 318 F F F 22 440-459 190 190 190 317 317 317 F F F 23 460-479 192 192 192 319 319 319 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 191 191 191 318 318 318 T T T 26 520-539 191 191 191 318 318 318 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 190 190 190 317 317 317 F F F 29 580-599 190 190 190 317 317 317 T T T Max gen: 194, Max flush: 324 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 325 325 325 F F F 1 020-039 195 195 195 325 325 325 F F F 2 040-059 195 195 195 325 325 325 F F F 3 060-079 194 194 194 323 323 323 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 195 195 195 325 325 325 F F F 6 120-139 195 195 195 325 325 325 F F F 7 140-159 195 195 195 325 325 325 F F F 8 160-179 195 195 195 325 325 325 F F F 9 180-199 193 193 193 321 321 321 F F F 10 200-219 193 193 193 321 321 321 F F F 11 220-239 194 194 194 324 324 324 F F F 12 240-259 194 194 194 323 323 323 F F F 13 260-279 191 191 191 318 318 318 F F F 14 280-299 194 194 194 324 324 324 F F F 15 300-319 194 194 194 323 323 323 F F F 16 320-339 192 192 192 320 320 320 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 F F F 20 400-419 183 183 183 307 307 307 F F F 21 420-439 191 191 191 318 318 318 F F F 22 440-459 190 190 190 317 317 317 F F F 23 460-479 192 192 192 319 319 319 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 195 195 195 325 325 325 F F F 26 520-539 191 191 191 318 318 318 F F F 27 540-559 184 184 184 308 308 308 F F F 28 560-579 190 190 190 317 317 317 F F F 29 580-599 195 195 195 325 325 325 F F F Max gen: 195, Max flush: 325 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.299572098Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.343131551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.343706427Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.343750807Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.343766676Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.343781516Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} {"msg":"Crucible 3db9d0e4-66dd-45ba-9c8d-c33c1bc7842f has session id: b93cd6a9-50b3-4dcd-9081-1082f5105452","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344314012Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Upstairs opts: Upstairs UUID: 3db9d0e4-66dd-45ba-9c8d-c33c1bc7842f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344356052Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Crucible stats registered with UUID: 3db9d0e4-66dd-45ba-9c8d-c33c1bc7842f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344380692Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:55.344404082Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344844628Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344877948Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} {"msg":"3db9d0e4-66dd-45ba-9c8d-c33c1bc7842f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344937158Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344964358Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"0","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.344992797Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"0","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345007057Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"1","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345021487Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"1","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345043317Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"2","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345072787Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"2","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345109977Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345137586Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345166596Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"ds_connection connected from Ok(127.0.0.1:34849)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345196566Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"ds_connection connected from Ok(127.0.0.1:55931)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345294325Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"ds_connection connected from Ok(127.0.0.1:38099)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.345373395Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34593447Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"2","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.3459684Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"0","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.346536306Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"2","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.346687805Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"0","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.347143551Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.347235361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"1","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34729828Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.347847006Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"1","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[0]R flush_numbers[0..12]: [325, 325, 325, 323, 320, 325, 325, 325, 325, 321, 321, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348351422Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[0]R generation[0..12]: [195, 195, 195, 194, 192, 195, 195, 195, 195, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348374652Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348390352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[1]R flush_numbers[0..12]: [325, 325, 325, 323, 320, 325, 325, 325, 325, 321, 321, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348409132Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[1]R generation[0..12]: [195, 195, 195, 194, 192, 195, 195, 195, 195, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348424382Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348439561Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[2]R flush_numbers[0..12]: [325, 325, 325, 323, 320, 325, 325, 325, 325, 321, 321, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348453412Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[2]R generation[0..12]: [195, 195, 195, 194, 192, 195, 195, 195, 195, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348466961Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348480811Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348493501Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348512751Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348529071Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348542071Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"":"downstairs","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.348555121Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34856702Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"0","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.3485804Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"1","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34859367Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"client":"2","":"downstairs","session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34860685Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2025-03-18T05:09:55.34862747Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"3db9d0e4-66dd-45ba-9c8d-c33c1bc7842f is now active with session: b93cd6a9-50b3-4dcd-9081-1082f5105452","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34864068Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34865241Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976,"session_id":"b93cd6a9-50b3-4dcd-9081-1082f5105452"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34866492Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} {"msg":"Activated sub_volume 3db9d0e4-66dd-45ba-9c8d-c33c1bc7842f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.34867603Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 190 len 20480 data: 13 13 12 11 10 02/30 Write block 178 len 12288 data: 13 13 13 03/30 Read block 11 len 40960 04/30 Read block 80 len 36864 05/30 Read block 319 len 36864 06/30 Write block 414 len 40960 data: 12 11 11 11 13 16 17 18 17 17 07/30 Read block 578 len 24576 08/30 Read block 167 len 28672 09/30 Flush 10/30 Write block 218 len 20480 data: 15 13 13 16 16 11/30 Write block 331 len 28672 data: 12 12 10 12 13 15 16 12/30 Write block 281 len 12288 data: 17 16 15 13/30 Read block 17 len 12288 14/30 Flush 15/30 Write block 232 len 20480 data: 11 11 12 15 14 16/30 Write block 285 len 24576 data: 20 17 20 20 19 21 17/30 Write block 178 len 36864 data: 14 14 14 14 16 16 19 17 17 18/30 Write block 542 len 12288 data: 13 12 15 19/30 Write block 149 len 16384 data: 20 19 19 21 20/30 Read block 499 len 16384 21/30 Read block 471 len 40960 22/30 Read block 381 len 40960 23/30 Read block 76 len 32768 24/30 Read block 357 len 12288 25/30 Read block 21 len 40960 26/30 Read block 302 len 32768 27/30 Read block 560 len 24576 28/30 Read block 140 len 28672 29/30 Read block 256 len 4096 30/30 Read block 13 len 8192 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:55.369903861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1976} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 325 325 325 F F F 1 020-039 195 195 195 325 325 325 F F F 2 040-059 195 195 195 325 325 325 F F F 3 060-079 194 194 194 323 323 323 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 195 195 195 325 325 325 F F F 6 120-139 195 195 195 325 325 325 F F F 7 140-159 195 195 195 325 325 325 T T T 8 160-179 196 196 196 326 326 326 T T T 9 180-199 196 196 196 326 326 326 T T T 10 200-219 196 196 196 327 327 327 F F F 11 220-239 196 196 196 327 327 327 T T T 12 240-259 194 194 194 323 323 323 F F F 13 260-279 191 191 191 318 318 318 F F F 14 280-299 196 196 196 327 327 327 T T T 15 300-319 194 194 194 323 323 323 F F F 16 320-339 196 196 196 327 327 327 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 F F F 20 400-419 196 196 196 326 326 326 F F F 21 420-439 196 196 196 326 326 326 F F F 22 440-459 190 190 190 317 317 317 F F F 23 460-479 192 192 192 319 319 319 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 195 195 195 325 325 325 F F F 26 520-539 191 191 191 318 318 318 F F F 27 540-559 184 184 184 308 308 308 T T T 28 560-579 190 190 190 317 317 317 F F F 29 580-599 195 195 195 325 325 325 F F F Max gen: 196, Max flush: 327 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 325 325 325 F F F 1 020-039 195 195 195 325 325 325 F F F 2 040-059 195 195 195 325 325 325 F F F 3 060-079 194 194 194 323 323 323 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 195 195 195 325 325 325 F F F 6 120-139 195 195 195 325 325 325 F F F 7 140-159 197 197 197 328 328 328 F F F 8 160-179 197 197 197 328 328 328 F F F 9 180-199 197 197 197 328 328 328 F F F 10 200-219 196 196 196 327 327 327 F F F 11 220-239 197 197 197 328 328 328 F F F 12 240-259 194 194 194 323 323 323 F F F 13 260-279 191 191 191 318 318 318 F F F 14 280-299 197 197 197 328 328 328 F F F 15 300-319 194 194 194 323 323 323 F F F 16 320-339 196 196 196 327 327 327 F F F 17 340-359 185 185 185 311 311 311 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 F F F 20 400-419 196 196 196 326 326 326 F F F 21 420-439 196 196 196 326 326 326 F F F 22 440-459 190 190 190 317 317 317 F F F 23 460-479 192 192 192 319 319 319 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 195 195 195 325 325 325 F F F 26 520-539 191 191 191 318 318 318 F F F 27 540-559 197 197 197 328 328 328 F F F 28 560-579 190 190 190 317 317 317 F F F 29 580-599 195 195 195 325 325 325 F F F Max gen: 197, Max flush: 328 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.61390144Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.657305115Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.657841701Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"045990dac130b2f233c4a41d8bf452c550cf571c\",\n git_commit_timestamp: \"2025-03-18T04:42:28.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65788361Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65789861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65791284Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} {"msg":"Crucible 725a80f5-9e02-4377-82f2-d7bedc33e5e6 has session id: f42b22c0-11c7-4940-a1bc-68c7f61689a1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.658468076Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Upstairs opts: Upstairs UUID: 725a80f5-9e02-4377-82f2-d7bedc33e5e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.658506506Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Crucible stats registered with UUID: 725a80f5-9e02-4377-82f2-d7bedc33e5e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.658530086Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:59.658550625Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.658980092Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659021732Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} {"msg":"725a80f5-9e02-4377-82f2-d7bedc33e5e6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659042822Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659056352Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"0","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659098781Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"0","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659122461Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"1","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659149251Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"1","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659177861Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"2","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659208271Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"2","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65923476Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65926294Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65928888Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52889)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65931508Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61994)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.65934235Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58652)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.659369349Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.660020814Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"0","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.660049924Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"1","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.660548321Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"1","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.66067004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"0","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.661240385Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.661276705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.661491883Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"2","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662055929Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"2","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[0]R flush_numbers[0..12]: [325, 325, 325, 323, 320, 325, 325, 328, 328, 328, 327, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662567705Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[0]R generation[0..12]: [195, 195, 195, 194, 192, 195, 195, 197, 197, 197, 196, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662588295Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662603255Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[1]R flush_numbers[0..12]: [325, 325, 325, 323, 320, 325, 325, 328, 328, 328, 327, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662617125Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[1]R generation[0..12]: [195, 195, 195, 194, 192, 195, 195, 197, 197, 197, 196, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662630665Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662644505Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[2]R flush_numbers[0..12]: [325, 325, 325, 323, 320, 325, 325, 328, 328, 328, 327, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662658165Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[2]R generation[0..12]: [195, 195, 195, 194, 192, 195, 195, 197, 197, 197, 196, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662671585Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662685274Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662702414Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662715944Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:09:59.662728964Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662746094Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"":"downstairs","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662758974Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662770914Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"0","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662784024Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"1","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662797114Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"client":"2","":"downstairs","session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662810853Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662823004Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"725a80f5-9e02-4377-82f2-d7bedc33e5e6 is now active with session: f42b22c0-11c7-4940-a1bc-68c7f61689a1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662835743Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662847443Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984,"session_id":"f42b22c0-11c7-4940-a1bc-68c7f61689a1"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662859583Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} {"msg":"Activated sub_volume 725a80f5-9e02-4377-82f2-d7bedc33e5e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:59.662870543Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 353 len 12288 data: 21 23 18 02/30 Read block 7 len 8192 03/30 Flush 04/30 Write block 453 len 8192 data: 9 9 05/30 Write block 544 len 12288 data: 16 18 20 06/30 Write block 21 len 4096 data: 23 07/30 Read block 165 len 40960 08/30 Write block 261 len 12288 data: 11 12 15 09/30 Write block 421 len 40960 data: 19 18 18 15 15 14 14 11 14 14 10/30 Write block 466 len 40960 data: 14 14 14 15 16 14 11 9 12 11 11/30 Write block 586 len 4096 data: 16 12/30 Write block 451 len 8192 data: 10 8 13/30 Read block 57 len 40960 14/30 Read block 424 len 16384 15/30 Read block 105 len 8192 16/30 Read block 494 len 20480 17/30 Flush 18/30 Write block 514 len 32768 data: 12 16 16 19 19 23 22 25 19/30 Write block 428 len 28672 data: 12 15 15 18 16 16 16 20/30 Read block 301 len 36864 21/30 Flush 22/30 Write block 509 len 32768 data: 13 16 15 12 16 13 17 17 23/30 Read block 271 len 16384 24/30 Write block 451 len 32768 data: 11 9 10 10 12 13 13 9 25/30 Read block 77 len 8192 26/30 Read block 163 len 24576 27/30 Read block 168 len 32768 28/30 Write block 396 len 12288 data: 8 8 9 29/30 Write block 556 len 20480 data: 19 20 17 15 16 30/30 Read block 172 len 16384 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 9 8 7 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:59.682397897Z","hostname":"w-01JPKSTB4XHMD5K3GSGPT6XHFX","pid":1984} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 325 325 325 F F F 1 020-039 198 198 198 330 330 330 F F F 2 040-059 195 195 195 325 325 325 F F F 3 060-079 194 194 194 323 323 323 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 195 195 195 325 325 325 F F F 6 120-139 195 195 195 325 325 325 F F F 7 140-159 197 197 197 328 328 328 F F F 8 160-179 197 197 197 328 328 328 F F F 9 180-199 197 197 197 328 328 328 F F F 10 200-219 196 196 196 327 327 327 F F F 11 220-239 197 197 197 328 328 328 F F F 12 240-259 194 194 194 323 323 323 F F F 13 260-279 198 198 198 330 330 330 F F F 14 280-299 197 197 197 328 328 328 F F F 15 300-319 194 194 194 323 323 323 F F F 16 320-339 196 196 196 327 327 327 F F F 17 340-359 198 198 198 329 329 329 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 191 191 191 318 318 318 T T T 20 400-419 196 196 196 326 326 326 F F F 21 420-439 198 198 198 331 331 331 F F F 22 440-459 198 198 198 330 330 330 T T T 23 460-479 198 198 198 330 330 330 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 198 198 198 331 331 331 T T T 26 520-539 198 198 198 331 331 331 F F F 27 540-559 198 198 198 330 330 330 T T T 28 560-579 190 190 190 317 317 317 T T T 29 580-599 198 198 198 330 330 330 F F F Max gen: 198, Max flush: 331 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 325 325 325 F F F 1 020-039 198 198 198 330 330 330 F F F 2 040-059 195 195 195 325 325 325 F F F 3 060-079 194 194 194 323 323 323 F F F 4 080-099 192 192 192 320 320 320 F F F 5 100-119 195 195 195 325 325 325 F F F 6 120-139 195 195 195 325 325 325 F F F 7 140-159 197 197 197 328 328 328 F F F 8 160-179 197 197 197 328 328 328 F F F 9 180-199 197 197 197 328 328 328 F F F 10 200-219 196 196 196 327 327 327 F F F 11 220-239 197 197 197 328 328 328 F F F 12 240-259 194 194 194 323 323 323 F F F 13 260-279 198 198 198 330 330 330 F F F 14 280-299 197 197 197 328 328 328 F F F 15 300-319 194 194 194 323 323 323 F F F 16 320-339 196 196 196 327 327 327 F F F 17 340-359 198 198 198 329 329 329 F F F 18 360-379 187 187 187 313 313 313 F F F 19 380-399 199 199 199 332 332 332 F F F 20 400-419 196 196 196 326 326 326 F F F 21 420-439 198 198 198 331 331 331 F F F 22 440-459 199 199 199 332 332 332 F F F 23 460-479 198 198 198 330 330 330 F F F 24 480-499 191 191 191 318 318 318 F F F 25 500-519 199 199 199 332 332 332 F F F 26 520-539 198 198 198 331 331 331 F F F 27 540-559 199 199 199 332 332 332 F F F 28 560-579 199 199 199 332 332 332 F F F 29 580-599 198 198 198 330 330 330 F F F Max gen: 199, Max flush: 332 11:03 Test duration Test completed real 11:03.635709375 user 49.215073923 sys 18.846296791 trap 0.069147093 tflt 0.090022752 dflt 0.127843330 kflt 0.005917083 lock 17:42:59.081300443 slp 54:31.054527158 lat 5.684676015 stop 0.061106729