Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.26511648 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.020251336 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020450272 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1193 1194 1195 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.427401496Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.480053989Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.481273007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.481375594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.481394164Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.481409923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198} {"msg":"Crucible a48980d1-4f49-49f2-97fd-f3c39e838b8e has session id: 2973e5a8-bdf4-46a6-a29a-d815e4e91462","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.482193693Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Upstairs opts: Upstairs UUID: a48980d1-4f49-49f2-97fd-f3c39e838b8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.482234062Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Crucible stats registered with UUID: a48980d1-4f49-49f2-97fd-f3c39e838b8e","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.482250951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:23.482265311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483088199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483136208Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198} {"msg":"a48980d1-4f49-49f2-97fd-f3c39e838b8e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483207886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483233965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"0","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483281524Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"0","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483299493Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"1","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483314393Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"1","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483327953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"2","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483343502Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"2","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483366602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483392211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483490488Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"ds_connection connected from Ok(127.0.0.1:41767)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483521718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"ds_connection connected from Ok(127.0.0.1:50028)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483550557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"ds_connection connected from Ok(127.0.0.1:42311)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.483764291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.485568354Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"1","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.485596153Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"2","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.485615512Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"0","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.486655895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"1","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.486683375Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"2","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.486698664Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"0","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487401405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487426525Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487518962Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487537892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487552101Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487565401Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487578321Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487591771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.48760479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.48761777Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487633919Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487646879Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487659659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487672088Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487684748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"":"downstairs","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487697628Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487709157Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"0","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487721927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"1","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487734617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"client":"2","":"downstairs","session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487754136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487777746Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"a48980d1-4f49-49f2-97fd-f3c39e838b8e is now active with session: 2973e5a8-bdf4-46a6-a29a-d815e4e91462","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487793435Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487814075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198,"session_id":"2973e5a8-bdf4-46a6-a29a-d815e4e91462"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487837184Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198} {"msg":"Activated sub_volume a48980d1-4f49-49f2-97fd-f3c39e838b8e","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:23.487859193Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1198} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.603066289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:27.646921393Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.647320473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.678877132Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.679505205Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.679538464Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.679552044Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.679566093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} {"msg":"Crucible 666cae2f-2424-40ed-9939-36e07d9c3cce has session id: 37c85dad-7aef-4db6-bc05-0d68f2cfd1f9","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680173157Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Upstairs opts: Upstairs UUID: 666cae2f-2424-40ed-9939-36e07d9c3cce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680208596Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Crucible stats registered with UUID: 666cae2f-2424-40ed-9939-36e07d9c3cce","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680227166Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:27.680239716Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680624435Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680648405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} {"msg":"666cae2f-2424-40ed-9939-36e07d9c3cce active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680706663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680723363Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"0","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680748322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"0","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680772691Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"1","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680799231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"1","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.68082525Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"2","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.6808522Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"2","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680886439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680914158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680940907Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40964)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680969546Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:36031)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.680999285Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50086)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.681167381Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.681654478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"1","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.681693987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"2","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.681837854Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"0","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.682370909Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"1","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.682405828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"2","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.682430268Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"0","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683018512Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683044682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.68308603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.68310291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.68311665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683129819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683146209Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683160848Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683174508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683187648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683201298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683213867Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683226547Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683239166Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683251776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"":"downstairs","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683264466Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683276105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"0","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683288985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"1","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683301625Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"client":"2","":"downstairs","session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683314575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-03-19T16:57:27.683335194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"666cae2f-2424-40ed-9939-36e07d9c3cce is now active with session: 37c85dad-7aef-4db6-bc05-0d68f2cfd1f9","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683347844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683361203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200,"session_id":"37c85dad-7aef-4db6-bc05-0d68f2cfd1f9"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683373123Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} {"msg":"Activated sub_volume 666cae2f-2424-40ed-9939-36e07d9c3cce","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:27.683383983Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 32 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Read block 130 len 32768 03/30 Read block 124 len 20480 04/30 Read block 254 len 40960 05/30 Write block 137 len 24576 data: 2 2 2 2 2 2 06/30 Read block 472 len 28672 07/30 Write block 446 len 12288 data: 2 2 2 08/30 Read block 318 len 28672 09/30 Write block 40 len 16384 data: 2 2 2 2 10/30 Write block 307 len 36864 data: 2 2 2 2 2 2 2 2 2 11/30 Write block 553 len 20480 data: 2 2 2 2 2 12/30 Write block 340 len 4096 data: 2 13/30 Read block 264 len 36864 14/30 Read block 271 len 16384 15/30 Write block 509 len 12288 data: 2 2 2 16/30 Write block 326 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Write block 195 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Write block 595 len 8192 data: 2 2 19/30 Write block 416 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Write block 211 len 8192 data: 2 2 21/30 Read block 79 len 16384 22/30 Write block 486 len 12288 data: 2 2 2 23/30 Read block 109 len 40960 24/30 Write block 380 len 40960 data: 2 2 2 2 2 2 2 2 2 2 25/30 Read block 579 len 20480 26/30 Read block 522 len 40960 27/30 Write block 74 len 16384 data: 2 2 2 2 28/30 Read block 509 len 8192 29/30 Read block 491 len 24576 30/30 Read block 539 len 28672 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:27.70434405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1200} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 T T T 2 040-059 1 1 1 1 1 1 T T T 3 060-079 1 1 1 1 1 1 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 T T T 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 T T T 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 T T T 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 T T T Max gen: 1, Max flush: 1 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 2 2 2 2 2 2 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 2 2 2 2 2 2 F F F 17 340-359 2 2 2 2 2 2 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 2, Max flush: 2 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.034097743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:32.076601424Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.076969444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.107836451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.108440075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.108475924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.108490384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.108504553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} {"msg":"Crucible 29636b5f-dbf3-48a7-a932-d2b29da9c4cc has session id: ffbe29f6-024a-4668-b2da-6a96e0ce6f09","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109071659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Upstairs opts: Upstairs UUID: 29636b5f-dbf3-48a7-a932-d2b29da9c4cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109115987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Crucible stats registered with UUID: 29636b5f-dbf3-48a7-a932-d2b29da9c4cc","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109139707Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:32.109159956Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109550666Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109573715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} {"msg":"29636b5f-dbf3-48a7-a932-d2b29da9c4cc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109637473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109655613Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"0","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109684672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"0","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109709032Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"1","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109737541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"1","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.10976282Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"2","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109789309Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"2","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109816019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109847578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109879517Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"ds_connection connected from Ok(127.0.0.1:53879)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.109898877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"ds_connection connected from Ok(127.0.0.1:42264)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.110094121Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"ds_connection connected from Ok(127.0.0.1:45723)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.11013896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.110733805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"1","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.11089694Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"2","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.111117165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"0","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.111382978Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"1","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.111571833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"2","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.111774277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"0","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.111954043Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112030821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[0]R flush_numbers[0..12]: [1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112349192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[0]R generation[0..12]: [1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112367272Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112381481Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[1]R flush_numbers[0..12]: [1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112394721Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[1]R generation[0..12]: [1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.11240778Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.11242147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[2]R flush_numbers[0..12]: [1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.11243445Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[2]R generation[0..12]: [1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112447599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112464629Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Max found gen is 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112477549Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Generation requested: 4 >= found:3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112490528Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112503138Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112515578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"":"downstairs","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112528457Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112540177Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"0","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112552987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"1","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":Wait for a query_work_queue command to finish before sending IO "Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112574476Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"client":"2","":"downstairs","session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112587376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112599266Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"29636b5f-dbf3-48a7-a932-d2b29da9c4cc is now active with session: ffbe29f6-024a-4668-b2da-6a96e0ce6f09","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112611605Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112623125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208,"session_id":"ffbe29f6-024a-4668-b2da-6a96e0ce6f09"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112634954Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} {"msg":"Activated sub_volume 29636b5f-dbf3-48a7-a932-d2b29da9c4cc","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:32.112645804Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 99 len 28672 data: 2 2 2 2 2 2 2 02/30 Write block 525 len 4096 data: 2 03/30 Write block 543 len 16384 data: 2 2 2 2 04/30 Read block 259 len 8192 05/30 Read block 554 len 12288 06/30 Read block 33 len 28672 07/30 Read block 65 len 12288 08/30 Write block 21 len 24576 data: 2 2 2 2 2 2 09/30 Read block 64 len 4096 10/30 Read block 525 len 40960 11/30 Write block 151 len 4096 data: 2 12/30 Write block 143 len 20480 data: 2 2 2 2 2 13/30 Write block 331 len 32768 data: 3 3 3 2 2 2 2 2 14/30 Read block 525 len 4096 15/30 Read block 44 len 40960 16/30 Write block 291 len 16384 data: 2 2 2 2 17/30 Write block 543 len 20480 data: 3 3 3 3 2 18/30 Flush 19/30 Write block 550 len 24576 data: 2 2 2 3 3 3 20/30 Read block 226 len 40960 21/30 Write block 341 len 12288 data: 2 2 2 22/30 Read block 332 len 40960 23/30 Flush 24/30 Write block 62 len 8192 data: 2 2 25/30 Write block 196 len 4096 data: 3 26/30 Read block 28 len 32768 27/30 Read block 102 len 36864 28/30 Read block 183 len 8192 29/30 Write block 427 len 20480 data: 2 2 2 2 2 30/30 Read block 163 len 40960 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:32.132480272Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 4 4 4 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 2 2 2 2 2 2 T T T 4 080-099 4 4 4 3 3 3 F F F 5 100-119 4 4 4 3 3 3 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 3 3 3 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 T T T 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 4 4 4 3 3 3 F F F 17 340-359 4 4 4 4 4 4 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 2 2 2 2 2 2 T T T 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 4 4 4 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 4, Max flush: 4 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 4 4 4 3 3 3 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 4 4 4 3 3 3 F F F 5 100-119 4 4 4 3 3 3 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 3 3 3 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 5 5 5 5 5 5 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 2 2 2 2 2 2 F F F 16 320-339 4 4 4 3 3 3 F F F 17 340-359 4 4 4 4 4 4 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 5 5 5 5 5 5 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 4 4 4 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 5, Max flush: 5 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.385632514Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.429657084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.430361636Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.430396224Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.430409764Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.430423974Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} {"msg":"Crucible 3ab5a69c-a09e-4e63-a1d0-eda8fd705a18 has session id: 4243ce28-a28c-4abf-9d80-4c094f3aecf3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431047137Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Upstairs opts: Upstairs UUID: 3ab5a69c-a09e-4e63-a1d0-eda8fd705a18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431083597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Crucible stats registered with UUID: 3ab5a69c-a09e-4e63-a1d0-eda8fd705a18","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431098426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:36.431110986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431527585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431549904Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} {"msg":"3ab5a69c-a09e-4e63-a1d0-eda8fd705a18 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431609472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431626002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"0","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431647551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"0","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431661171Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"1","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431675211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"1","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.43169765Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"2","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431724129Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"2","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431738069Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431751459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431764739Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:61020)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431832397Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:52420)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431864066Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48770)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.431890135Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.432586747Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"2","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.432624326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"0","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.432745623Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"1","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.43322918Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"0","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.433365036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"1","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.433397765Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"2","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.433792275Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.433915812Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[0]R flush_numbers[0..12]: [1, 3, 2, 5, 3, 3, 2, 3, 1, 5, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434054738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[0]R generation[0..12]: [1, 4, 2, 5, 4, 4, 2, 4, 1, 5, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434074828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434089557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[1]R flush_numbers[0..12]: [1, 3, 2, 5, 3, 3, 2, 3, 1, 5, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434106357Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[1]R generation[0..12]: [1, 4, 2, 5, 4, 4, 2, 4, 1, 5, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434120046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434133696Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[2]R flush_numbers[0..12]: [1, 3, 2, 5, 3, 3, 2, 3, 1, 5, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434146856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[2]R generation[0..12]: [1, 4, 2, 5, 4, 4, 2, 4, 1, 5, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434159865Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434173515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434186155Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434199004Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:57:36.434211704Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434230583Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"":"downstairs","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434243303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434254913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"0","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434267793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"1","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434280482Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"client":"2","":"downstairs","session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434293292Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434307012Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"3ab5a69c-a09e-4e63-a1d0-eda8fd705a18 is now active with session: 4243ce28-a28c-4abf-9d80-4c094f3aecf3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434319661Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434331281Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216,"session_id":"4243ce28-a28c-4abf-9d80-4c094f3aecf3"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.434343051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} {"msg":"Activated sub_volume 3ab5a69c-a09e-4e63-a1d0-eda8fd705a18","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:36.43435394Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 24576 data: 2 2 2 2 2 2 02/30 Write block 323 len 4096 data: 2 03/30 Read block 112 len 28672 04/30 Write block 294 len 12288 data: 3 2 2 05/30 Flush 06/30 Write block 118 len 4096 data: 2 07/30 Read block 419 len 8192 08/30 Write block 226 len 32768 data: 2 2 2 2 2 2 2 2 09/30 Read block 305 len 4096 10/30 Read block 499 len 4096 11/30 Write block 145 len 4096 data: 3 12/30 Write block 117 len 16384 data: 2 3 2 2 13/30 Write block 311 len 20480 data: 3 3 3 3 3 14/30 Read block 395 len 8192 15/30 Write block 343 len 24576 data: 3 2 2 2 2 2 16/30 Write block 209 len 8192 data: 2 2 17/30 Read block 492 len 32768 18/30 Read block 533 len 36864 19/30 Write block 302 len 12288 data: 2 2 2 20/30 Read block 356 len 32768 21/30 Write block 473 len 36864 data: 2 2 2 2 2 2 2 2 2 22/30 Read block 228 len 40960 23/30 Flush 24/30 Write block 455 len 4096 data: 2 25/30 Read block 21 len 36864 26/30 Read block 45 len 40960 27/30 Read block 35 len 36864 28/30 Read block 84 len 4096 29/30 Write block 298 len 12288 data: 2 2 2 30/30 Write block 25 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:36.453176274Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 4 4 4 3 3 3 T T T 2 040-059 2 2 2 2 2 2 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 4 4 4 3 3 3 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 5 5 5 5 5 5 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 6 6 6 6 6 6 T T T 15 300-319 6 6 6 7 7 7 T T T 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 5 5 5 5 5 5 F F F 22 440-459 2 2 2 2 2 2 T T T 23 460-479 6 6 6 7 7 7 F F F 24 480-499 6 6 6 7 7 7 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 4 4 4 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 6, Max flush: 7 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 4 4 4 3 3 3 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 5 5 5 5 5 5 F F F 10 200-219 6 6 6 7 7 7 F F F 11 220-239 6 6 6 7 7 7 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 7 7 7 8 8 8 F F F 15 300-319 7 7 7 8 8 8 F F F 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 5 5 5 5 5 5 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 6 6 6 7 7 7 F F F 24 480-499 6 6 6 7 7 7 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 4 4 4 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 7, Max flush: 8 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.677105007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.720725608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.72141084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.721455229Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.721474278Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.721489278Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} {"msg":"Crucible d32c8df5-3d63-41ff-9e90-4a80e2b33457 has session id: cf7a6c09-dcb9-4a2a-b832-83a10de07f76","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722052223Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Upstairs opts: Upstairs UUID: d32c8df5-3d63-41ff-9e90-4a80e2b33457, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722096622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Crucible stats registered with UUID: d32c8df5-3d63-41ff-9e90-4a80e2b33457","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722119771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:40.722133451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.72255675Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722580209Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} {"msg":"d32c8df5-3d63-41ff-9e90-4a80e2b33457 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722656217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722672787Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"0","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722695186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"0","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722712196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"1","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722736895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"1","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722762134Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"2","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722793923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"2","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722819573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722845932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722872541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"ds_connection connected from Ok(127.0.0.1:55108)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.722898881Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"ds_connection connected from Ok(127.0.0.1:59183)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.72292526Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"ds_connection connected from Ok(127.0.0.1:51126)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.72295123Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.72368426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"2","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.723712019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"0","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.724193927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"0","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.724298634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"2","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.724659824Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.724884439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.725665408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"1","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726247603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"1","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[0]R flush_numbers[0..12]: [1, 8, 2, 5, 3, 7, 7, 7, 1, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726794328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[0]R generation[0..12]: [1, 7, 2, 5, 4, 6, 6, 6, 1, 5, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726817257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726831607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[1]R flush_numbers[0..12]: [1, 8, 2, 5, 3, 7, 7, 7, 1, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726844927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[1]R generation[0..12]: [1, 7, 2, 5, 4, 6, 6, 6, 1, 5, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726857936Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726871426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[2]R flush_numbers[0..12]: [1, 8, 2, 5, 3, 7, 7, 7, 1, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726884356Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[2]R generation[0..12]: [1, 7, 2, 5, 4, 6, 6, 6, 1, 5, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726897466Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726911125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726923855Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726936625Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726948994Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:57:40.726961374Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"":"downstairs","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726983423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.726995503Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"0","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.727008243Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"1","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.727020942Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"client":"2","":"downstairs","session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.727033622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.727045371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"d32c8df5-3d63-41ff-9e90-4a80e2b33457 is now active with session: cf7a6c09-dcb9-4a2a-b832-83a10de07f76","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.727057711Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.727069361Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224,"session_id":"cf7a6c09-dcb9-4a2a-b832-83a10de07f76"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.727081241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} {"msg":"Activated sub_volume d32c8df5-3d63-41ff-9e90-4a80e2b33457","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:40.72709218Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 16384 data: 3 3 3 2 02/30 Read block 443 len 32768 03/30 Flush 04/30 Write block 221 len 28672 data: 2 2 2 2 2 3 3 05/30 Read block 446 len 24576 06/30 Read block 370 len 12288 07/30 Read block 588 len 20480 08/30 Read block 265 len 20480 09/30 Read block 310 len 40960 10/30 Flush 11/30 Write block 370 len 20480 data: 2 2 2 2 2 12/30 Write block 47 len 16384 data: 2 2 2 2 13/30 Flush 14/30 Write block 239 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Read block 182 len 32768 16/30 Write block 96 len 32768 data: 2 2 2 3 3 3 3 3 17/30 Write block 85 len 20480 data: 2 2 2 2 2 18/30 Write block 470 len 4096 data: 2 19/30 Read block 594 len 24576 20/30 Write block 277 len 32768 data: 2 2 2 2 2 2 2 2 21/30 Read block 260 len 20480 22/30 Write block 395 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Read block 140 len 32768 24/30 Write block 83 len 28672 data: 2 2 3 3 3 3 3 25/30 Read block 366 len 32768 26/30 Read block 165 len 28672 27/30 Write block 547 len 16384 data: 3 2 2 3 28/30 Write block 235 len 16384 data: 2 2 2 2 29/30 Write block 250 len 4096 data: 2 30/30 Write block 431 len 16384 data: 3 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 4 Sent Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:40.747298038Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 8 8 8 11 11 11 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 4 4 4 3 3 3 T T T 5 100-119 6 6 6 7 7 7 T T T 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 5 5 5 5 5 5 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 8 8 8 10 10 10 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 1 1 1 1 1 1 T T T 14 280-299 7 7 7 8 8 8 T T T 15 300-319 7 7 7 8 8 8 F F F 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 8 8 8 11 11 11 F F F 19 380-399 2 2 2 2 2 2 T T T 20 400-419 2 2 2 2 2 2 T T T 21 420-439 5 5 5 5 5 5 T T T 22 440-459 7 7 7 8 8 8 F F F 23 460-479 6 6 6 7 7 7 T T T 24 480-499 6 6 6 7 7 7 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 4 4 4 4 4 4 T T T 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 8, Max flush: 11 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 8 8 8 11 11 11 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 9 9 9 12 12 12 F F F 5 100-119 9 9 9 12 12 12 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 5 5 5 5 5 5 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 9 9 9 12 12 12 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 9 9 9 12 12 12 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 7 7 7 8 8 8 F F F 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 8 8 8 11 11 11 F F F 19 380-399 9 9 9 12 12 12 F F F 20 400-419 9 9 9 12 12 12 F F F 21 420-439 9 9 9 12 12 12 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 9 9 9 12 12 12 F F F 24 480-499 6 6 6 7 7 7 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 9 9 9 12 12 12 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 9, Max flush: 12 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.031556572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.074814532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.075430906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.075466995Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.075481134Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.075495184Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} {"msg":"Crucible 77ff4008-16a3-41d7-9f6b-dcf6413a057d has session id: 08f988f7-91ab-44cd-93d9-4949e2791b74","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076111078Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Upstairs opts: Upstairs UUID: 77ff4008-16a3-41d7-9f6b-dcf6413a057d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076156726Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Crucible stats registered with UUID: 77ff4008-16a3-41d7-9f6b-dcf6413a057d","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076183446Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:55.076199365Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076665783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076689672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} {"msg":"77ff4008-16a3-41d7-9f6b-dcf6413a057d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076761001Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.07677859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"0","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076802469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"0","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076815949Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"1","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076829799Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"1","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076842708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076856538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076869348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076882727Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.076896037Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:38881)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.077054773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:55.077097572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:62294)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.077127061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:55.077208609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:57:55.077245728Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.077266237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.077659127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"0","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.077696956Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"1","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.078373558Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"1","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.078402927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"0","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.078967923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:57:55.07906346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.078458558Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"ds_connection connected from Ok(127.0.0.1:57859)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.078690002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.079558339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080149233Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[0]R flush_numbers[0..12]: [1, 8, 11, 5, 12, 12, 7, 7, 1, 5, 9, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080742198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[0]R generation[0..12]: [1, 7, 8, 5, 9, 9, 6, 6, 1, 5, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080778277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080794966Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[1]R flush_numbers[0..12]: [1, 8, 11, 5, 12, 12, 7, 7, 1, 5, 9, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080809076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[1]R generation[0..12]: [1, 7, 8, 5, 9, 9, 6, 6, 1, 5, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080822826Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080836685Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[2]R flush_numbers[0..12]: [1, 8, 11, 5, 12, 12, 7, 7, 1, 5, 9, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080849975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[2]R generation[0..12]: [1, 7, 8, 5, 9, 9, 6, 6, 1, 5, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080863105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080876644Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080889224Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080902144Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080914643Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080927273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"":"downstairs","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080940363Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080952172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"0","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080965502Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"1","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080980302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"client":"2","":"downstairs","session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.080993721Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.081005631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"77ff4008-16a3-41d7-9f6b-dcf6413a057d is now active with session: 08f988f7-91ab-44cd-93d9-4949e2791b74","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.08101807Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.08102987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232,"session_id":"08f988f7-91ab-44cd-93d9-4949e2791b74"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.08104194Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} {"msg":"Activated sub_volume 77ff4008-16a3-41d7-9f6b-dcf6413a057d","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:05.081059279Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 40960 data: 2 2 2 2 3 3 3 3 3 3 02/30 Write block 398 len 32768 data: 3 3 3 3 3 2 2 2 03/30 Read block 243 len 20480 04/30 Write block 124 len 8192 data: 2 2 05/30 Write block 162 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Read block 91 len 24576 07/30 Read block 476 len 36864 08/30 Write block 227 len 20480 data: 4 3 3 3 3 09/30 Flush 10/30 Write block 170 len 16384 data: 3 2 2 2 11/30 Read block 170 len 20480 12/30 Write block 477 len 20480 data: 3 3 3 3 3 13/30 Write block 235 len 12288 data: 3 3 3 14/30 Write block 415 len 12288 data: 2 3 3 15/30 Read block 313 len 8192 16/30 Read block 450 len 28672 17/30 Write block 199 len 16384 data: 3 3 3 3 18/30 Read block 570 len 36864 19/30 Write block 418 len 16384 data: 3 3 3 3 20/30 Read block 194 len 8192 21/30 Read block 99 len 20480 22/30 Write block 100 len 16384 data: 4 4 4 4 23/30 Read block 497 len 36864 24/30 Write block 28 len 36864 data: 3 3 3 3 4 4 4 4 4 25/30 Read block 117 len 32768 26/30 Write block 170 len 12288 data: 4 3 3 27/30 Write block 494 len 20480 data: 2 2 2 2 2 28/30 Read block 230 len 32768 29/30 Write block 391 len 32768 data: 2 2 2 2 3 3 3 4 30/30 Read block 341 len 20480 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 4 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 21 20 19 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:05.101323906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 10 10 10 13 13 13 T T T 2 040-059 8 8 8 11 11 11 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 9 9 9 12 12 12 F F F 5 100-119 9 9 9 12 12 12 T T T 6 120-139 10 10 10 13 13 13 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 10 10 10 13 13 13 T T T 9 180-199 5 5 5 5 5 5 T T T 10 200-219 8 8 8 9 9 9 T T T 11 220-239 10 10 10 13 13 13 T T T 12 240-259 9 9 9 12 12 12 F F F 13 260-279 9 9 9 12 12 12 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 7 7 7 8 8 8 F F F 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 8 8 8 11 11 11 F F F 19 380-399 10 10 10 13 13 13 T T T 20 400-419 10 10 10 13 13 13 T T T 21 420-439 9 9 9 12 12 12 T T T 22 440-459 7 7 7 8 8 8 F F F 23 460-479 9 9 9 12 12 12 T T T 24 480-499 6 6 6 7 7 7 T T T 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 9 9 9 12 12 12 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 10, Max flush: 13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 11 11 11 14 14 14 F F F 2 040-059 8 8 8 11 11 11 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 9 9 9 12 12 12 F F F 5 100-119 11 11 11 14 14 14 F F F 6 120-139 10 10 10 13 13 13 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 11 11 11 14 14 14 F F F 9 180-199 11 11 11 14 14 14 F F F 10 200-219 11 11 11 14 14 14 F F F 11 220-239 11 11 11 14 14 14 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 9 9 9 12 12 12 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 7 7 7 8 8 8 F F F 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 8 8 8 11 11 11 F F F 19 380-399 11 11 11 14 14 14 F F F 20 400-419 11 11 11 14 14 14 F F F 21 420-439 11 11 11 14 14 14 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 11 11 11 14 14 14 F F F 24 480-499 11 11 11 14 14 14 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 9 9 9 12 12 12 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 11, Max flush: 14 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.364764328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:09.407250348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.407646858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.438426377Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.438993182Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.439027771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.439041231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.43905542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} {"msg":"Crucible 62b8850f-6448-4c70-b5f2-559c9c8751c2 has session id: bc943d6d-9285-4660-8710-458ac693af69","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.439696423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Upstairs opts: Upstairs UUID: 62b8850f-6448-4c70-b5f2-559c9c8751c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.439727753Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Crucible stats registered with UUID: 62b8850f-6448-4c70-b5f2-559c9c8751c2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.439742342Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:09.439757872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440179831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.44020258Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} {"msg":"62b8850f-6448-4c70-b5f2-559c9c8751c2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440261128Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440277858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"0","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440300487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"0","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440314197Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"1","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440328267Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"1","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440341246Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"2","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440391835Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"2","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440429744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440457203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440482783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"ds_connection connected from Ok(127.0.0.1:39253)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440509792Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"ds_connection connected from Ok(127.0.0.1:58879)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.440538801Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"ds_connection connected from Ok(127.0.0.1:35831)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.44056842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.441218603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"2","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.441248622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"1","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.441434498Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"0","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.441870936Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"2","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.441899276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"1","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442175098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"0","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442443491Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442553428Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[0]R flush_numbers[0..12]: [1, 14, 11, 5, 12, 14, 13, 7, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442668845Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[0]R generation[0..12]: [1, 11, 8, 5, 9, 11, 10, 6, 11, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442689245Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442703904Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[1]R flush_numbers[0..12]: [1, 14, 11, 5, 12, 14, 13, 7, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442718134Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[1]R generation[0..12]: [1, 11, 8, 5, 9, 11, 10, 6, 11, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442731623Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442745373Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[2]R flush_numbers[0..12]: [1, 14, 11, 5, 12, 14, 13, 7, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442761913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[2]R generation[0..12]: [1, 11, 8, 5, 9, 11, 10, 6, 11, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442775472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442789422Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442801902Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442814641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442827301Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69Wait for a query_work_queue command to finish before sending IO "} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.44284656Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"":"downstairs","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.44285939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.44287112Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"0","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442883959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"1","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442896779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"client":"2","":"downstairs","session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442909439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442921349Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"62b8850f-6448-4c70-b5f2-559c9c8751c2 is now active with session: bc943d6d-9285-4660-8710-458ac693af69","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442933998Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442945648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240,"session_id":"bc943d6d-9285-4660-8710-458ac693af69"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442957488Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} {"msg":"Activated sub_volume 62b8850f-6448-4c70-b5f2-559c9c8751c2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:09.442970067Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 28672 data: 2 2 2 2 2 3 3 02/30 Write block 551 len 36864 data: 3 3 4 4 4 3 3 2 2 03/30 Read block 147 len 28672 04/30 Write block 493 len 4096 data: 2 05/30 Read block 406 len 20480 06/30 Flush 07/30 Write block 360 len 4096 data: 2 08/30 Read block 520 len 4096 09/30 Read block 483 len 16384 10/30 Write block 96 len 32768 data: 3 3 3 4 5 5 5 5 11/30 Write block 272 len 16384 data: 2 2 2 2 12/30 Read block 289 len 8192 13/30 Write block 103 len 12288 data: 6 3 3 14/30 Write block 11 len 36864 data: 2 2 2 2 2 2 2 2 2 15/30 Read block 299 len 12288 16/30 Read block 426 len 24576 17/30 Read block 364 len 12288 18/30 Read block 192 len 24576 19/30 Write block 430 len 12288 data: 3 4 3 20/30 Read block 223 len 16384 21/30 Read block 98 len 32768 22/30 Read block 67 len 32768 23/30 Read block 377 len 16384 24/30 Read block 114 len 16384 25/30 Read block 203 len 4096 26/30 Read block 222 len 16384 27/30 Read block 125 len 36864 28/30 Write block 435 len 12288 data: 2 2 2 29/30 Read block 33 len 40960 30/30 Write block 309 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 23 22 22 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:09.462499453Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 11 11 11 14 14 14 F F F 2 040-059 8 8 8 11 11 11 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 9 9 9 12 12 12 T T T 5 100-119 11 11 11 14 14 14 T T T 6 120-139 12 12 12 15 15 15 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 11 11 11 14 14 14 F F F 9 180-199 11 11 11 14 14 14 F F F 10 200-219 11 11 11 14 14 14 F F F 11 220-239 11 11 11 14 14 14 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 9 9 9 12 12 12 T T T 14 280-299 9 9 9 12 12 12 F F F 15 300-319 7 7 7 8 8 8 T T T 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 8 8 8 11 11 11 T T T 19 380-399 11 11 11 14 14 14 F F F 20 400-419 11 11 11 14 14 14 F F F 21 420-439 11 11 11 14 14 14 T T T 22 440-459 7 7 7 8 8 8 F F F 23 460-479 11 11 11 14 14 14 F F F 24 480-499 12 12 12 15 15 15 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 12 12 12 15 15 15 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 12, Max flush: 15 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 16 16 16 F F F 1 020-039 11 11 11 14 14 14 F F F 2 040-059 8 8 8 11 11 11 F F F 3 060-079 5 5 5 5 5 5 F F F 4 080-099 13 13 13 16 16 16 F F F 5 100-119 13 13 13 16 16 16 F F F 6 120-139 12 12 12 15 15 15 F F F 7 140-159 6 6 6 7 7 7 F F F 8 160-179 11 11 11 14 14 14 F F F 9 180-199 11 11 11 14 14 14 F F F 10 200-219 11 11 11 14 14 14 F F F 11 220-239 11 11 11 14 14 14 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 13 13 13 16 16 16 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 13 13 13 16 16 16 F F F 16 320-339 6 6 6 6 6 6 F F F 17 340-359 6 6 6 7 7 7 F F F 18 360-379 13 13 13 16 16 16 F F F 19 380-399 11 11 11 14 14 14 F F F 20 400-419 11 11 11 14 14 14 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 11 11 11 14 14 14 F F F 24 480-499 12 12 12 15 15 15 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 12 12 12 15 15 15 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 13, Max flush: 16 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.736565615Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:13.778748543Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.779131533Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.810079437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.810680901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81071599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81073005Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81074431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} {"msg":"Crucible b54c672a-56f5-4b10-9ebc-38e1d819746e has session id: 16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811325334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Upstairs opts: Upstairs UUID: b54c672a-56f5-4b10-9ebc-38e1d819746e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811368893Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Crucible stats registered with UUID: b54c672a-56f5-4b10-9ebc-38e1d819746e","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811394532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:13.811416052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811832021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81186769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} {"msg":"b54c672a-56f5-4b10-9ebc-38e1d819746e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811924438Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811950588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"0","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811975237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"0","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.811997977Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"1","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812025236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"1","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812051225Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"2","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812077534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"2","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812103914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812129763Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812155072Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51824)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812181212Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:49814)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812210391Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:65324)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812432365Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812963441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"1","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.812993501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"2","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81302165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"0","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.813674463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"1","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.813701772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"0","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.813724191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"2","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814224128Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814326585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[0]R flush_numbers[0..12]: [16, 14, 11, 5, 16, 16, 15, 7, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814396613Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[0]R generation[0..12]: [13, 11, 8, 5, 13, 13, 12, 6, 11, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814415923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814430172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[1]R flush_numbers[0..12]: [16, 14, 11, 5, 16, 16, 15, 7, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814443692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[1]R generation[0..12]: [13, 11, 8, 5, 13, 13, 12, 6, 11, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814456772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814470242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[2]R flush_numbers[0..12]: [16, 14, 11, 5, 16, 16, 15, 7, 14, 14, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814483301Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[2]R generation[0..12]: [13, 11, 8, 5, 13, 13, 12, 6, 11, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814496421Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81451007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81452286Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.81453554Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814551179Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814564139Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"":"downstairs","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814577019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814588998Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"0","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814601988Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"1","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814614748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"client":"2","":"downstairs","session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814627417Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814639377Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"b54c672a-56f5-4b10-9ebc-38e1d819746e is now active with session: 16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814651717Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msgWait for a query_work_queue command to finish before sending IO ":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814669796Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248,"session_id":"16e987fd-3a64-4ec8-acdb-f0bb8f0d6bf9"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814681806Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} {"msg":"Activated sub_volume b54c672a-56f5-4b10-9ebc-38e1d819746e","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:13.814692856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Flush 04/30 Flush 05/30 Write block 89 len 4096 data: 4 06/30 Write block 214 len 12288 data: 2 2 2 07/30 Read block 385 len 8192 08/30 Read block 340 len 28672 09/30 Flush 10/30 Write block 123 len 12288 data: 2 3 3 11/30 Flush 12/30 Write block 560 len 28672 data: 2 2 2 2 2 2 2 13/30 Write block 44 len 4096 data: 2 14/30 Read block 170 len 32768 15/30 Flush 16/30 Write block 62 len 32768 data: 3 3 2 2 2 2 2 2 17/30 Flush 18/30 Write block 352 len 20480 data: 2 2 2 2 2 19/30 Read block 129 len 24576 20/30 Read block 264 len 36864 21/30 Write block 334 len 28672 data: 3 3 3 3 3 2 3 22/30 Write block 18 len 20480 data: 3 3 2 3 3 23/30 Flush 24/30 Flush 25/30 Write block 317 len 4096 data: 2 26/30 Write block 155 len 4096 data: 2 27/30 Write block 348 len 16384 data: 3 2 2 2 28/30 Write block 541 len 20480 data: 2 2 4 4 4 29/30 Read block 104 len 16384 30/30 Write block 358 len 24576 data: 2 2 3 2 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:13.832784769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 14 14 14 25 25 25 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 13 13 13 16 16 16 F F F 6 120-139 14 14 14 22 22 22 F F F 7 140-159 6 6 6 7 7 7 T T T 8 160-179 11 11 11 14 14 14 F F F 9 180-199 11 11 11 14 14 14 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 11 11 11 14 14 14 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 13 13 13 16 16 16 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 13 13 13 16 16 16 T T T 16 320-339 14 14 14 25 25 25 F F F 17 340-359 14 14 14 25 25 25 T T T 18 360-379 13 13 13 16 16 16 T T T 19 380-399 11 11 11 14 14 14 F F F 20 400-419 11 11 11 14 14 14 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 11 11 11 14 14 14 F F F 24 480-499 12 12 12 15 15 15 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 12 12 12 15 15 15 T T T 28 560-579 14 14 14 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 14, Max flush: 25 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 14 14 14 25 25 25 F F F 2 040-059 14 14 14 23 23 23 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 13 13 13 16 16 16 F F F 6 120-139 14 14 14 22 22 22 F F F 7 140-159 15 15 15 26 26 26 F F F 8 160-179 11 11 11 14 14 14 F F F 9 180-199 11 11 11 14 14 14 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 11 11 11 14 14 14 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 13 13 13 16 16 16 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 15 15 15 26 26 26 F F F 16 320-339 14 14 14 25 25 25 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 15 15 15 26 26 26 F F F 19 380-399 11 11 11 14 14 14 F F F 20 400-419 11 11 11 14 14 14 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 11 11 11 14 14 14 F F F 24 480-499 12 12 12 15 15 15 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 15, Max flush: 26 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.092748662Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.139342375Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.140045886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.140081685Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.140096045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.140110345Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} {"msg":"Crucible 7d9be5f3-6ea8-4f20-a750-3403577c5baa has session id: 81c64419-53f7-44c2-a127-7d6dbbca197e","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.140610341Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Upstairs opts: Upstairs UUID: 7d9be5f3-6ea8-4f20-a750-3403577c5baa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.14066842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Crucible stats registered with UUID: 7d9be5f3-6ea8-4f20-a750-3403577c5baa","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.140694599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:18.140717599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141254084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141280994Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} {"msg":"7d9be5f3-6ea8-4f20-a750-3403577c5baa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141338032Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141354872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"0","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141377391Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"0","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141391121Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"1","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.14140514Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"1","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.1414269Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"2","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141457449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"2","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141485348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141511768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141539387Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34894)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141668133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50883)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141703733Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56894)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.141731332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.142350206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"0","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.1425655Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"1","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.143043317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"0","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.143185933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"1","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.143566434Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.143649081Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"2","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.143755509Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144151508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"2","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[0]R flush_numbers[0..12]: [25, 25, 23, 24, 21, 16, 22, 26, 14, 14, 21, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144664415Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[0]R generation[0..12]: [14, 14, 14, 14, 14, 13, 14, 15, 11, 11, 14, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144684074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144698764Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[1]R flush_numbers[0..12]: [25, 25, 23, 24, 21, 16, 22, 26, 14, 14, 21, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144712483Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[1]R generation[0..12]: [14, 14, 14, 14, 14, 13, 14, 15, 11, 11, 14, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144725993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144739523Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[2]R flush_numbers[0..12]: [25, 25, 23, 24, 21, 16, 22, 26, 14, 14, 21, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144752842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[2]R generation[0..12]: [14, 14, 14, 14, 14, 13, 14, 15, 11, 11, 14, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144769462Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144783371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144796131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144809951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144873689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144891969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"":"downstairs","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144905428Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144917118Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"0","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144930608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"1","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144944197Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"client":"2","":"downstairs","session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144957377Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144969687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"7d9be5f3-6ea8-4f20-a750-3403577c5baa is now active with session: 81c64419-53f7-44c2-a127-7d6dbbca197e","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144982566Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.144994346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256,"session_id":"81c64419-53f7-44c2-a127-7d6dbbca197e"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.145006506Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} {"msg":"Activated sub_volume 7d9be5f3-6ea8-4f20-a750-3403577c5baa","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:18.145017685Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 127 len 40960 data: 2 2 2 2 2 3 3 3 3 3 02/30 Read block 528 len 36864 03/30 Read block 405 len 24576 04/30 Read block 37 len 24576 05/30 Write block 319 len 24576 data: 2 2 2 2 3 2 06/30 Read block 371 len 8192 07/30 Read block 314 len 8192 08/30 Read block 506 len 32768 09/30 Flush 10/30 Write block 183 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Read block 424 len 4096 12/30 Write block 469 len 32768 data: 2 3 2 2 3 3 3 3 13/30 Read block 433 len 8192 14/30 Read block 577 len 40960 15/30 Write block 53 len 8192 data: 2 2 16/30 Read block 209 len 20480 17/30 Read block 233 len 40960 18/30 Write block 89 len 20480 data: 5 2 2 2 2 19/30 Flush 20/30 Write block 125 len 4096 data: 4 21/30 Read block 229 len 4096 22/30 Flush 23/30 Write block 111 len 12288 data: 2 2 2 24/30 Flush 25/30 Write block 511 len 16384 data: 3 2 2 3 26/30 Read block 82 len 4096 27/30 Read block 530 len 12288 28/30 Read block 323 len 36864 29/30 Read block 273 len 8192 30/30 Read block 226 len 8192 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:18.166013952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 14 14 14 25 25 25 F F F 2 040-059 16 16 16 28 28 28 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 16 16 16 29 29 29 F F F 7 140-159 15 15 15 26 26 26 F F F 8 160-179 11 11 11 14 14 14 F F F 9 180-199 16 16 16 28 28 28 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 11 11 11 14 14 14 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 13 13 13 16 16 16 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 16 16 16 27 27 27 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 15 15 15 26 26 26 F F F 19 380-399 11 11 11 14 14 14 F F F 20 400-419 11 11 11 14 14 14 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 16 16 16 28 28 28 F F F 24 480-499 12 12 12 15 15 15 F F F 25 500-519 6 6 6 6 6 6 T T T 26 520-539 4 4 4 3 3 3 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 16, Max flush: 30 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 14 14 14 25 25 25 F F F 2 040-059 16 16 16 28 28 28 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 16 16 16 29 29 29 F F F 7 140-159 15 15 15 26 26 26 F F F 8 160-179 11 11 11 14 14 14 F F F 9 180-199 16 16 16 28 28 28 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 11 11 11 14 14 14 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 13 13 13 16 16 16 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 16 16 16 27 27 27 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 15 15 15 26 26 26 F F F 19 380-399 11 11 11 14 14 14 F F F 20 400-419 11 11 11 14 14 14 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 7 7 7 8 8 8 F F F 23 460-479 16 16 16 28 28 28 F F F 24 480-499 12 12 12 15 15 15 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 17, Max flush: 31 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.381937125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.425506347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.426244898Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.426284217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.426298436Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.426312376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} {"msg":"Crucible 640c1b7f-2bc6-4c1e-8293-c671d9ee9e81 has session id: 4a6ee02b-e185-4736-9f9a-f923a682bd20","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.426837672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Upstairs opts: Upstairs UUID: 640c1b7f-2bc6-4c1e-8293-c671d9ee9e81, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.426885901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Crucible stats registered with UUID: 640c1b7f-2bc6-4c1e-8293-c671d9ee9e81","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.42690919Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:22.42692259Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427440146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427464206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} {"msg":"640c1b7f-2bc6-4c1e-8293-c671d9ee9e81 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427538234Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427558773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"0","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427595842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"0","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427626552Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"1","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.42765834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"1","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427721499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427758408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427781217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427805957Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427834986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:22.427869535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:22.427900364Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"ds_connection connected from Ok(127.0.0.1:52705)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.427929313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:22.428026661Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"ds_connection connected from Ok(127.0.0.1:48117)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.4280494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.428077409Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.428478249Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"0","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.428635295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"1","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.429103082Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"0","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.429238449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"1","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.429670687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:22.429752556Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.429596091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"ds_connection connected from Ok(127.0.0.1:33000)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.429745677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.430743951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.431389074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[0]R flush_numbers[0..12]: [25, 25, 28, 24, 28, 30, 29, 26, 14, 28, 21, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432017217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[0]R generation[0..12]: [14, 14, 16, 14, 16, 16, 16, 15, 11, 16, 14, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432045577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432062346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[1]R flush_numbers[0..12]: [25, 25, 28, 24, 28, 30, 29, 26, 14, 28, 21, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432076656Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[1]R generation[0..12]: [14, 14, 16, 14, 16, 16, 16, 15, 11, 16, 14, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432090166Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432103785Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[2]R flush_numbers[0..12]: [25, 25, 28, 24, 28, 30, 29, 26, 14, 28, 21, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432117155Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[2]R generation[0..12]: [14, 14, 16, 14, 16, 16, 16, 15, 11, 16, 14, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432130494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432144204Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432156914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432169724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432182413Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432198063Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"":"downstairs","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432211383Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432223172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"0","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432236272Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"1","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432249071Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"client":"2","":"downstairs","session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432262101Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432274051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"640c1b7f-2bc6-4c1e-8293-c671d9ee9e81 is now active with session: 4a6ee02b-e185-4736-9f9a-f923a682bd20","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432286461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.43229807Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264,"session_id":"4a6ee02b-e185-4736-9f9a-f923a682bd20"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.43231004Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} {"msg":"Activated sub_volume 640c1b7f-2bc6-4c1e-8293-c671d9ee9e81","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:32.432320929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 20480 data: 3 3 3 3 4 02/30 Write block 410 len 16384 data: 2 2 2 2 03/30 Write block 224 len 32768 data: 3 3 4 5 4 4 4 4 04/30 Write block 394 len 4096 data: 4 05/30 Read block 56 len 36864 06/30 Flush 07/30 Write block 6 len 8192 data: 2 2 08/30 Write block 190 len 12288 data: 3 3 3 09/30 Read block 183 len 24576 10/30 Read block 412 len 24576 11/30 Read block 572 len 40960 12/30 Write block 271 len 4096 data: 2 13/30 Write block 268 len 36864 data: 2 2 2 3 3 3 3 3 2 14/30 Write block 451 len 16384 data: 2 2 2 2 15/30 Read block 415 len 40960 16/30 Write block 137 len 28672 data: 4 4 3 3 3 3 3 17/30 Read block 7 len 20480 18/30 Write block 360 len 4096 data: 4 19/30 Read block 106 len 24576 20/30 Read block 128 len 20480 21/30 Read block 298 len 8192 22/30 Read block 101 len 20480 23/30 Write block 6 len 40960 data: 3 3 2 2 2 3 3 3 3 3 24/30 Read block 21 len 8192 25/30 Read block 158 len 36864 26/30 Write block 121 len 12288 data: 2 2 3 27/30 Read block 57 len 40960 28/30 Read block 238 len 36864 29/30 Write block 466 len 4096 data: 2 30/30 Write block 486 len 24576 data: 3 3 3 2 2 2 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 2 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:32.451609241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 T T T 1 020-039 14 14 14 25 25 25 F F F 2 040-059 16 16 16 28 28 28 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 16 16 16 29 29 29 T T T 7 140-159 15 15 15 26 26 26 T T T 8 160-179 11 11 11 14 14 14 F F F 9 180-199 16 16 16 28 28 28 T T T 10 200-219 14 14 14 21 21 21 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 13 13 13 16 16 16 T T T 14 280-299 9 9 9 12 12 12 F F F 15 300-319 16 16 16 27 27 27 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 15 15 15 26 26 26 T T T 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 7 7 7 8 8 8 T T T 23 460-479 16 16 16 28 28 28 T T T 24 480-499 12 12 12 15 15 15 T T T 25 500-519 17 17 17 31 31 31 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 18, Max flush: 32 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 33 33 33 F F F 1 020-039 14 14 14 25 25 25 F F F 2 040-059 16 16 16 28 28 28 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 19 19 19 33 33 33 F F F 8 160-179 11 11 11 14 14 14 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 19 19 19 33 33 33 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 16 16 16 27 27 27 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 19 19 19 33 33 33 F F F 19 380-399 18 18 18 32 32 32 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 19 19 19 33 33 33 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 19 19 19 33 33 33 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 4 4 4 3 3 3 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 19, Max flush: 33 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.709409502Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.753201999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.75389007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.7539232Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.753942239Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.753957079Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} {"msg":"Crucible d209dd50-85b4-499f-b285-c866d7b3aeea has session id: 32ee25ec-abc5-45ed-b404-596a786e70b8","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.754533163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Upstairs opts: Upstairs UUID: d209dd50-85b4-499f-b285-c866d7b3aeea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.754565823Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Crucible stats registered with UUID: d209dd50-85b4-499f-b285-c866d7b3aeea","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.754583852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:46.754604291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755072489Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755095018Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} {"msg":"d209dd50-85b4-499f-b285-c866d7b3aeea active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755167567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755184776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"0","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755207526Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"0","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755221075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"1","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755234765Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"1","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755247525Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755261204Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755273784Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755287024Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755300273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:46.7554125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48012)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755457569Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:50741)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755485828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:46.755514687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:46.755544597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.755565496Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.756084332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"0","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.756112832Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"1","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.756648458Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"1","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.756757975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"0","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.757224302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:46.75731416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.756871536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:59989)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.757040511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.758171651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.758811175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[0]R flush_numbers[0..12]: [33, 25, 28, 24, 28, 30, 33, 33, 14, 33, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759444038Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[0]R generation[0..12]: [19, 14, 16, 14, 16, 16, 19, 19, 11, 19, 14, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759471837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759488217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[1]R flush_numbers[0..12]: [33, 25, 28, 24, 28, 30, 33, 33, 14, 33, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759502556Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[1]R generation[0..12]: [19, 14, 16, 14, 16, 16, 19, 19, 11, 19, 14, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759515996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759529686Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[2]R flush_numbers[0..12]: [33, 25, 28, 24, 28, 30, 33, 33, 14, 33, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759542765Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[2]R generation[0..12]: [19, 14, 16, 14, 16, 16, 19, 19, 11, 19, 14, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759556075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759569725Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759582794Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759595914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759608334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759620783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"":"downstairs","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759647343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759672452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"0","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759698311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"1","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759722991Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"client":"2","":"downstairs","session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.75974536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.75976928Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"d209dd50-85b4-499f-b285-c866d7b3aeea is now active with session: 32ee25ec-abc5-45ed-b404-596a786e70b8","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759800519Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759818098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272,"session_id":"32ee25ec-abc5-45ed-b404-596a786e70b8"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759831618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} {"msg":"Activated sub_volume d209dd50-85b4-499f-b285-c866d7b3aeea","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:58:56.759850637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 445 len 8192 data: 2 3 03/30 Write block 362 len 28672 data: 3 3 2 2 2 2 2 04/30 Write block 39 len 28672 data: 3 3 3 3 3 3 2 05/30 Write block 362 len 12288 data: 4 4 3 06/30 Read block 19 len 28672 07/30 Write block 178 len 8192 data: 2 2 08/30 Read block 221 len 8192 09/30 Read block 271 len 40960 10/30 Read block 231 len 16384 11/30 Write block 247 len 4096 data: 3 12/30 Read block 436 len 4096 13/30 Write block 524 len 36864 data: 2 3 2 2 2 2 2 2 2 14/30 Write block 161 len 32768 data: 2 3 3 3 3 3 3 3 15/30 Read block 193 len 32768 16/30 Flush 17/30 Write block 376 len 36864 data: 2 2 2 2 3 3 3 3 3 18/30 Read block 205 len 28672 19/30 Write block 148 len 20480 data: 2 2 2 3 2 20/30 Write block 259 len 16384 data: 2 2 2 2 21/30 Write block 164 len 24576 data: 4 4 4 4 4 3 22/30 Write block 480 len 4096 data: 4 23/30 Read block 381 len 12288 24/30 Write block 252 len 40960 data: 2 2 2 2 2 2 2 3 3 3 25/30 Flush 26/30 Flush 27/30 Write block 575 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Write block 450 len 40960 data: 2 3 3 3 3 3 2 2 2 2 29/30 Read block 13 len 24576 30/30 Write block 91 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:58:56.780291328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 33 33 33 F F F 1 020-039 20 20 20 35 35 35 F F F 2 040-059 20 20 20 35 35 35 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 16 16 16 28 28 28 T T T 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 20 20 20 36 36 36 F F F 8 160-179 20 20 20 36 36 36 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 20 20 20 36 36 36 F F F 13 260-279 20 20 20 36 36 36 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 16 16 16 27 27 27 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 20 20 20 36 36 36 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 20 20 20 35 35 35 T T T 23 460-479 19 19 19 33 33 33 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 20 20 20 35 35 35 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 14 14 14 23 23 23 T T T 29 580-599 2 2 2 2 2 2 T T T Max gen: 20, Max flush: 36 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 33 33 33 F F F 1 020-039 20 20 20 35 35 35 F F F 2 040-059 20 20 20 35 35 35 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 21 21 21 37 37 37 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 20 20 20 36 36 36 F F F 8 160-179 20 20 20 36 36 36 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 18 18 18 32 32 32 F F F 12 240-259 20 20 20 36 36 36 F F F 13 260-279 20 20 20 36 36 36 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 16 16 16 27 27 27 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 20 20 20 36 36 36 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 13 13 13 16 16 16 F F F 22 440-459 21 21 21 37 37 37 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 20 20 20 36 36 36 F F F 25 500-519 17 17 17 31 31 31 F F F 26 520-539 20 20 20 35 35 35 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 21 21 21 37 37 37 F F F 29 580-599 21 21 21 37 37 37 F F F Max gen: 21, Max flush: 37 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.020126901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:01.063007391Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.063544207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.095117975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.096032351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.09608065Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.096095459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.096109829Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} {"msg":"Crucible bbbe883d-ed38-4cc0-a1ce-3a76b80a65c0 has session id: b005eed0-ce83-49fc-9ad7-036396f06cf1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.096718473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Upstairs opts: Upstairs UUID: bbbe883d-ed38-4cc0-a1ce-3a76b80a65c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.096753682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Crucible stats registered with UUID: bbbe883d-ed38-4cc0-a1ce-3a76b80a65c0","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.096772862Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:01.096795551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.09721856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097243589Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} {"msg":"bbbe883d-ed38-4cc0-a1ce-3a76b80a65c0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097309987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097327497Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"0","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097351906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"0","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097366546Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"1","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097380555Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"1","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097393665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"2","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097407865Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"2","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097420994Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097434964Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097450573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33171)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.09757522Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58598)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097620499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57291)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.097650308Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.098494956Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"2","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.098525075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"1","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.09871382Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"0","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.099199498Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"2","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.099227687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"1","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.099409602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"0","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.099834571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.0998597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[0]R flush_numbers[0..12]: [33, 35, 35, 24, 37, 30, 33, 36, 36, 33, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100053335Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[0]R generation[0..12]: [19, 20, 20, 14, 21, 16, 19, 20, 20, 19, 14, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100081764Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100107534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[1]R flush_numbers[0..12]: [33, 35, 35, 24, 37, 30, 33, 36, 36, 33, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100133173Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[1]R generation[0..12]: [19, 20, 20, 14, 21, 16, 19, 20, 20, 19, 14, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100160152Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100185252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[2]R flush_numbers[0..12]: [33, 35, 35, 24, 37, 30, 33, 36, 36, 33, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100215761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[2]R generation[0..12]: [19, 20, 20, 14, 21, 16, 19, 20, 20, 19, 14, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.10024384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.10026127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100277569Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:59:01.100301018Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100326918Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100351317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"":"downstairs","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100368977Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100381126Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"0","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100400006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"1","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100423935Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"client":"2","":"downstairs","session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100449454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100468824Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"bbbe883d-ed38-4cc0-a1ce-3a76b80a65c0 is now active with session: b005eed0-ce83-49fc-9ad7-036396f06cf1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100481944Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100499043Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280,"session_id":"b005eed0-ce83-49fc-9ad7-036396f06cf1"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100520953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} {"msg":"Activated sub_volume bbbe883d-ed38-4cc0-a1ce-3a76b80a65c0","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:01.100546052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 312 len 12288 data: 4 4 4 03/30 Read block 378 len 20480 04/30 Flush 05/30 Write block 242 len 8192 data: 3 3 06/30 Write block 587 len 8192 data: 2 2 07/30 Write block 233 len 20480 data: 3 2 4 4 4 08/30 Read block 145 len 32768 09/30 Write block 162 len 4096 data: 4 10/30 Write block 511 len 28672 data: 4 3 3 4 3 3 3 11/30 Write block 451 len 24576 data: 4 4 4 4 4 3 12/30 Write block 441 len 4096 data: 2 13/30 Read block 497 len 20480 14/30 Write block 50 len 24576 data: 3 2 2 3 3 2 15/30 Read block 530 len 24576 16/30 Write block 55 len 12288 data: 3 2 2 17/30 Write block 428 len 4096 data: 3 18/30 Write block 494 len 12288 data: 3 3 3 19/30 Write block 27 len 16384 data: 2 4 4 4 20/30 Read block 453 len 12288 21/30 Flush 22/30 Write block 434 len 8192 data: 3 3 23/30 Write block 155 len 24576 data: 3 2 2 2 2 2 24/30 Read block 130 len 40960 25/30 Read block 29 len 36864 26/30 Write block 542 len 16384 data: 3 5 5 5 27/30 Write block 45 len 40960 data: 3 2 3 3 3 4 3 3 4 4 28/30 Read block 589 len 32768 29/30 Read block 565 len 32768 30/30 Read block 166 len 24576 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:01.120853617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 33 33 33 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 22 22 22 40 40 40 T T T 3 060-079 14 14 14 24 24 24 F F F 4 080-099 21 21 21 37 37 37 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 20 20 20 36 36 36 T T T 8 160-179 22 22 22 40 40 40 T T T 9 180-199 19 19 19 33 33 33 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 20 20 20 36 36 36 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 22 22 22 39 39 39 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 20 20 20 36 36 36 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 22 22 22 40 40 40 T T T 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 22 22 22 40 40 40 F F F 26 520-539 20 20 20 35 35 35 F F F 27 540-559 15 15 15 26 26 26 T T T 28 560-579 21 21 21 37 37 37 F F F 29 580-599 22 22 22 40 40 40 F F F Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 33 33 33 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 14 14 14 24 24 24 F F F 4 080-099 21 21 21 37 37 37 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 23 23 23 41 41 41 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 20 20 20 36 36 36 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 22 22 22 39 39 39 F F F 16 320-339 16 16 16 27 27 27 F F F 17 340-359 15 15 15 26 26 26 F F F 18 360-379 20 20 20 36 36 36 F F F 19 380-399 20 20 20 36 36 36 F F F 20 400-419 18 18 18 32 32 32 F F F 21 420-439 23 23 23 41 41 41 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 22 22 22 40 40 40 F F F 26 520-539 20 20 20 35 35 35 F F F 27 540-559 23 23 23 41 41 41 F F F 28 560-579 21 21 21 37 37 37 F F F 29 580-599 22 22 22 40 40 40 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.390692379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:05.432888898Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.433272008Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.464365468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.465004972Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.465040831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.46505872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.46507344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} {"msg":"Crucible 0c220d04-8685-4f8e-84de-de8063628da1 has session id: 1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.465696664Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Upstairs opts: Upstairs UUID: 0c220d04-8685-4f8e-84de-de8063628da1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.465732772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Crucible stats registered with UUID: 0c220d04-8685-4f8e-84de-de8063628da1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.465749222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:05.465762112Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466167511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466191091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} {"msg":"0c220d04-8685-4f8e-84de-de8063628da1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466262429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466279858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"0","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466303768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"0","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466317677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"1","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466331877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"1","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466345316Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"2","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466363266Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"2","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466376475Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466390395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466403945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"ds_connection connected from Ok(127.0.0.1:46748)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466428514Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"ds_connection connected from Ok(127.0.0.1:49373)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.4665738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"ds_connection connected from Ok(127.0.0.1:51275)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.466630749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.467199694Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"2","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.467233593Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"1","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.467487936Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"0","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.467994463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"1","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468025662Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"2","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468042612Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"0","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468629716Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468655026Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[0]R flush_numbers[0..12]: [33, 40, 41, 24, 37, 30, 33, 41, 41, 33, 21, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468673045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[0]R generation[0..12]: [19, 22, 23, 14, 21, 16, 19, 23, 23, 19, 14, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468688025Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468702144Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[1]R flush_numbers[0..12]: [33, 40, 41, 24, 37, 30, 33, 41, 41, 33, 21, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468715754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[1]R generation[0..12]: [19, 22, 23, 14, 21, 16, 19, 23, 23, 19, 14, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468729493Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468743263Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[2]R flush_numbers[0..12]: [33, 40, 41, 24, 37, 30, 33, 41, 41, 33, 21, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468756503Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[2]R generation[0..12]: [19, 22, 23, 14, 21, 16, 19, 23, 23, 19, 14, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468769563Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468783162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468795862Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468808502Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468820971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468833671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"":"downstairs","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.46884669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:59:05.46886039Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"0","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.46888056Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"1","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468893459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"client":"2","":"downstairs","session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468906519Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468918609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"0c220d04-8685-4f8e-84de-de8063628da1 is now active with session: 1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468931128Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468942728Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288,"session_id":"1c4a77b6-e2cb-4765-83a1-d4eaf7e84d90"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468954478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} {"msg":"Activated sub_volume 0c220d04-8685-4f8e-84de-de8063628da1","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:05.468965437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 173 len 16384 data: 3 2 2 2 02/30 Read block 144 len 32768 03/30 Write block 306 len 4096 data: 2 04/30 Write block 547 len 28672 data: 4 3 3 4 4 4 5 05/30 Write block 222 len 12288 data: 3 3 4 06/30 Read block 94 len 32768 07/30 Write block 348 len 20480 data: 4 3 3 3 3 08/30 Write block 543 len 36864 data: 6 6 6 4 5 4 4 5 5 09/30 Flush 10/30 Flush 11/30 Write block 250 len 4096 data: 3 12/30 Flush 13/30 Write block 171 len 40960 data: 4 4 4 3 3 3 2 3 3 2 14/30 Write block 63 len 36864 data: 4 3 3 3 3 3 3 2 2 15/30 Write block 569 len 16384 data: 2 2 2 2 16/30 Read block 542 len 8192 17/30 Read block 529 len 32768 18/30 Flush 19/30 Write block 259 len 16384 data: 4 4 4 3 20/30 Read block 54 len 36864 21/30 Flush 22/30 Write block 316 len 24576 data: 2 3 2 3 3 3 23/30 Write block 69 len 12288 data: 4 3 3 24/30 Read block 402 len 32768 25/30 Read block 542 len 16384 26/30 Write block 418 len 36864 data: 4 4 4 4 3 3 3 3 2 27/30 Write block 378 len 40960 data: 3 3 4 4 4 4 4 3 3 3 28/30 Read block 277 len 36864 29/30 Read block 322 len 20480 30/30 Write block 503 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:05.488820004Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 33 33 33 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 24 24 24 45 45 45 T T T 4 080-099 21 21 21 37 37 37 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 45 45 45 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 46 46 46 F F F 13 260-279 24 24 24 46 46 46 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 24 24 24 42 42 42 T T T 16 320-339 16 16 16 27 27 27 T T T 17 340-359 24 24 24 42 42 42 F F F 18 360-379 20 20 20 36 36 36 T T T 19 380-399 20 20 20 36 36 36 T T T 20 400-419 18 18 18 32 32 32 T T T 21 420-439 23 23 23 41 41 41 T T T 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 22 22 22 40 40 40 T T T 26 520-539 20 20 20 35 35 35 F F F 27 540-559 24 24 24 42 42 42 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 22 22 22 40 40 40 F F F Max gen: 24, Max flush: 46 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 33 33 33 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 25 25 25 47 47 47 F F F 4 080-099 21 21 21 37 37 37 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 45 45 45 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 46 46 46 F F F 13 260-279 24 24 24 46 46 46 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 25 25 25 47 47 47 F F F 16 320-339 25 25 25 47 47 47 F F F 17 340-359 24 24 24 42 42 42 F F F 18 360-379 25 25 25 47 47 47 F F F 19 380-399 25 25 25 47 47 47 F F F 20 400-419 25 25 25 47 47 47 F F F 21 420-439 25 25 25 47 47 47 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 25 25 25 47 47 47 F F F 26 520-539 20 20 20 35 35 35 F F F 27 540-559 24 24 24 42 42 42 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 22 22 22 40 40 40 F F F Max gen: 25, Max flush: 47 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.766394913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.809228143Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.809902176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.809951915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.809967034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.809981274Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} {"msg":"Crucible 27b61847-c964-4ee2-be06-b6c5653a17cc has session id: f03374f8-54b9-4cc0-882d-45f827ac27cc","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.810598367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Upstairs opts: Upstairs UUID: 27b61847-c964-4ee2-be06-b6c5653a17cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.810644736Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Crucible stats registered with UUID: 27b61847-c964-4ee2-be06-b6c5653a17cc","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.810668836Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:09.810690175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811083965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811109714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} {"msg":"27b61847-c964-4ee2-be06-b6c5653a17cc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811193452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811214731Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"0","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811237781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"0","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.81125132Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.81126516Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811287699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"2","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811312329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"2","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811326618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811349398Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811375627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:42887)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811475904Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:09.811515183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:50820)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811597551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:09.81162899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:09.811710248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.811743897Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.812212985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"2","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.812241154Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"0","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.812824939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"0","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.812852288Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"2","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.813457962Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:09.813483382Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.813422346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:40389)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.813588162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.814581966Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815195749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[0]R flush_numbers[0..12]: [33, 40, 41, 47, 37, 30, 33, 41, 45, 45, 21, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815798414Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[0]R generation[0..12]: [19, 22, 23, 25, 21, 16, 19, 23, 24, 24, 14, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815827133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815843732Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[1]R flush_numbers[0..12]: [33, 40, 41, 47, 37, 30, 33, 41, 45, 45, 21, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815863162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[1]R generation[0..12]: [19, 22, 23, 25, 21, 16, 19, 23, 24, 24, 14, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815877501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815891641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[2]R flush_numbers[0..12]: [33, 40, 41, 47, 37, 30, 33, 41, 45, 45, 21, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815905051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[2]R generation[0..12]: [19, 22, 23, 25, 21, 16, 19, 23, 24, 24, 14, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.81591828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.81593202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815944979Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815958259Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815971019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815983849Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"":"downstairs","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.815996788Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816008708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"0","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816021797Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"1","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816034617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"client":"2","":"downstairs","session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:59:19.816047917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816067256Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"27b61847-c964-4ee2-be06-b6c5653a17cc is now active with session: f03374f8-54b9-4cc0-882d-45f827ac27cc","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816080146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816091816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296,"session_id":"f03374f8-54b9-4cc0-882d-45f827ac27cc"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816103695Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} {"msg":"Activated sub_volume 27b61847-c964-4ee2-be06-b6c5653a17cc","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:19.816114735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 40 len 24576 data: 4 4 4 4 4 4 04/30 Write block 326 len 16384 data: 3 3 3 3 05/30 Write block 412 len 36864 data: 3 3 2 3 4 4 5 5 5 06/30 Flush 07/30 Write block 37 len 12288 data: 4 3 4 08/30 Read block 500 len 8192 09/30 Read block 445 len 32768 10/30 Read block 167 len 8192 11/30 Write block 17 len 16384 data: 3 4 4 3 12/30 Write block 480 len 28672 data: 5 4 2 2 2 2 4 13/30 Flush 14/30 Write block 226 len 12288 data: 5 6 5 15/30 Read block 122 len 4096 16/30 Read block 300 len 12288 17/30 Write block 423 len 8192 data: 4 4 18/30 Write block 26 len 16384 data: 3 3 5 5 19/30 Flush 20/30 Write block 338 len 32768 data: 4 3 4 3 3 4 3 3 21/30 Read block 303 len 28672 22/30 Flush 23/30 Write block 390 len 20480 data: 2 4 4 4 5 24/30 Flush 25/30 Write block 47 len 8192 data: 4 4 26/30 Flush 27/30 Write block 539 len 40960 data: 2 2 3 4 7 7 7 5 6 5 28/30 Read block 90 len 40960 29/30 Write block 331 len 32768 data: 4 4 4 4 4 4 4 5 30/30 Write block 238 len 24576 data: 3 3 3 3 4 4 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:19.836566196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 26 26 26 52 52 52 F F F 2 040-059 26 26 26 55 55 55 F F F 3 060-079 25 25 25 47 47 47 F F F 4 080-099 21 21 21 37 37 37 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 45 45 45 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 26 26 26 52 52 52 T T T 12 240-259 24 24 24 46 46 46 T T T 13 260-279 24 24 24 46 46 46 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 25 25 25 47 47 47 F F F 16 320-339 26 26 26 53 53 53 T T T 17 340-359 26 26 26 53 53 53 F F F 18 360-379 25 25 25 47 47 47 F F F 19 380-399 26 26 26 54 54 54 F F F 20 400-419 26 26 26 50 50 50 F F F 21 420-439 26 26 26 52 52 52 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 25 25 25 47 47 47 F F F 26 520-539 20 20 20 35 35 35 T T T 27 540-559 24 24 24 42 42 42 T T T 28 560-579 24 24 24 45 45 45 F F F 29 580-599 22 22 22 40 40 40 F F F Max gen: 26, Max flush: 55 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 26 26 26 52 52 52 F F F 2 040-059 26 26 26 55 55 55 F F F 3 060-079 25 25 25 47 47 47 F F F 4 080-099 21 21 21 37 37 37 F F F 5 100-119 16 16 16 30 30 30 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 45 45 45 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 14 14 14 21 21 21 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 27 27 27 56 56 56 F F F 13 260-279 24 24 24 46 46 46 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 25 25 25 47 47 47 F F F 16 320-339 27 27 27 56 56 56 F F F 17 340-359 26 26 26 53 53 53 F F F 18 360-379 25 25 25 47 47 47 F F F 19 380-399 26 26 26 54 54 54 F F F 20 400-419 26 26 26 50 50 50 F F F 21 420-439 26 26 26 52 52 52 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 25 25 25 47 47 47 F F F 26 520-539 27 27 27 56 56 56 F F F 27 540-559 27 27 27 56 56 56 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 22 22 22 40 40 40 F F F Max gen: 27, Max flush: 56 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.07182597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.115235556Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.115933617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.115969097Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.115987096Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.116001496Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} {"msg":"Crucible 48dca7e1-9c48-4a34-9f1a-c93e26db4ab0 has session id: 5d3ae049-be04-4096-bcf8-303d3bc4d3e2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.1165999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Upstairs opts: Upstairs UUID: 48dca7e1-9c48-4a34-9f1a-c93e26db4ab0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.116649119Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Crucible stats registered with UUID: 48dca7e1-9c48-4a34-9f1a-c93e26db4ab0","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.116674278Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:24.116687808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117155095Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117179515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} {"msg":"48dca7e1-9c48-4a34-9f1a-c93e26db4ab0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117244893Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117263033Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"0","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117287112Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"0","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117300661Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"1","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117314641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"1","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117327821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"2","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.11734571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"2","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.11735916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.11737464Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117399319Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40012)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117490776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47545)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117524876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49477)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.117681871Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.118208838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"0","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.118235967Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"2","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.118910189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"0","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.118950418Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"2","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.119583472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.119609581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.119839555Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"1","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120361451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"1","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[0]R flush_numbers[0..12]: [51, 52, 55, 47, 37, 30, 33, 41, 45, 45, 21, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120869127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[0]R generation[0..12]: [26, 26, 26, 25, 21, 16, 19, 23, 24, 24, 14, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120892767Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120907317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[1]R flush_numbers[0..12]: [51, 52, 55, 47, 37, 30, 33, 41, 45, 45, 21, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120920776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[1]R generation[0..12]: [26, 26, 26, 25, 21, 16, 19, 23, 24, 24, 14, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120934016Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120947645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[2]R flush_numbers[0..12]: [51, 52, 55, 47, 37, 30, 33, 41, 45, 45, 21, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120961355Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[2]R generation[0..12]: [26, 26, 26, 25, 21, 16, 19, 23, 24, 24, 14, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120974575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.120988124Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121000624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121013354Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121025774Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121038143Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"":"downstairs","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121050703Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:59:24.121065692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"0","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121083522Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"1","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121096372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"client":"2","":"downstairs","session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121109131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.121120971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"48dca7e1-9c48-4a34-9f1a-c93e26db4ab0 is now active with session: 5d3ae049-be04-4096-bcf8-303d3bc4d3e2","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.12113366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.12114533Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304,"session_id":"5d3ae049-be04-4096-bcf8-303d3bc4d3e2"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.12115732Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} {"msg":"Activated sub_volume 48dca7e1-9c48-4a34-9f1a-c93e26db4ab0","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:24.12116816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 200 len 32768 data: 4 4 4 3 3 2 2 2 02/30 Read block 48 len 40960 03/30 Write block 412 len 36864 data: 4 4 3 4 5 5 6 6 6 04/30 Read block 293 len 16384 05/30 Write block 211 len 12288 data: 4 4 3 06/30 Read block 338 len 12288 07/30 Write block 419 len 12288 data: 7 7 5 08/30 Read block 216 len 4096 09/30 Write block 587 len 4096 data: 3 10/30 Flush 11/30 Write block 34 len 20480 data: 5 5 5 5 4 12/30 Read block 515 len 12288 13/30 Read block 245 len 4096 14/30 Write block 508 len 28672 data: 2 3 3 5 4 4 5 15/30 Write block 568 len 4096 data: 2 16/30 Read block 100 len 4096 17/30 Write block 243 len 24576 data: 5 3 3 3 4 3 18/30 Read block 171 len 12288 19/30 Write block 293 len 20480 data: 3 4 3 3 2 20/30 Read block 431 len 40960 21/30 Write block 93 len 24576 data: 3 2 2 4 4 4 22/30 Write block 285 len 4096 data: 2 23/30 Flush 24/30 Write block 103 len 36864 data: 7 4 4 2 2 2 2 2 3 25/30 Flush 26/30 Write block 261 len 40960 data: 5 4 2 2 2 2 2 3 3 3 27/30 Write block 458 len 4096 data: 3 28/30 Write block 361 len 8192 data: 3 5 29/30 Read block 170 len 4096 30/30 Read block 152 len 40960 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:24.140480161Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 28 28 28 58 58 58 F F F 2 040-059 26 26 26 55 55 55 F F F 3 060-079 25 25 25 47 47 47 F F F 4 080-099 28 28 28 58 58 58 F F F 5 100-119 28 28 28 59 59 59 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 45 45 45 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 28 28 28 57 57 57 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 24 24 24 46 46 46 T T T 14 280-299 28 28 28 58 58 58 F F F 15 300-319 25 25 25 47 47 47 F F F 16 320-339 27 27 27 56 56 56 F F F 17 340-359 26 26 26 53 53 53 F F F 18 360-379 25 25 25 47 47 47 T T T 19 380-399 26 26 26 54 54 54 F F F 20 400-419 28 28 28 57 57 57 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 22 22 22 40 40 40 T T T 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 28 28 28 58 58 58 F F F 26 520-539 27 27 27 56 56 56 F F F 27 540-559 27 27 27 56 56 56 F F F 28 560-579 28 28 28 58 58 58 F F F 29 580-599 28 28 28 57 57 57 F F F Max gen: 28, Max flush: 59 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 28 28 28 58 58 58 F F F 2 040-059 26 26 26 55 55 55 F F F 3 060-079 25 25 25 47 47 47 F F F 4 080-099 28 28 28 58 58 58 F F F 5 100-119 28 28 28 59 59 59 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 45 45 45 F F F 9 180-199 24 24 24 45 45 45 F F F 10 200-219 28 28 28 57 57 57 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 29 29 29 60 60 60 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 25 25 25 47 47 47 F F F 16 320-339 27 27 27 56 56 56 F F F 17 340-359 26 26 26 53 53 53 F F F 18 360-379 29 29 29 60 60 60 F F F 19 380-399 26 26 26 54 54 54 F F F 20 400-419 28 28 28 57 57 57 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 29 29 29 60 60 60 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 28 28 28 58 58 58 F F F 26 520-539 27 27 27 56 56 56 F F F 27 540-559 27 27 27 56 56 56 F F F 28 560-579 28 28 28 58 58 58 F F F 29 580-599 28 28 28 57 57 57 F F F Max gen: 29, Max flush: 60 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.363035059Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:28.406148163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.40666801Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.439072206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.439846456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.439882375Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.439896564Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.439911454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} {"msg":"Crucible f61c1587-44ed-4749-b2e7-c902adc498c7 has session id: f4c804e6-a639-425b-b00a-08df4938cdbb","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.44042056Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Upstairs opts: Upstairs UUID: f61c1587-44ed-4749-b2e7-c902adc498c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.440569237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Crucible stats registered with UUID: f61c1587-44ed-4749-b2e7-c902adc498c7","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.440606505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:28.440628715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.440983646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441018985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} {"msg":"f61c1587-44ed-4749-b2e7-c902adc498c7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441065063Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441080423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"0","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441111772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"0","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441138531Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"1","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441164561Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"1","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.44119353Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"2","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441222089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"2","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441252228Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441279488Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441306347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39243)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441331586Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55499)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441433864Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"ds_connection connected from Ok(127.0.0.1:33071)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.441471543Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.44197466Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"2","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.442019798Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"1","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.442219073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"0","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.442641452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"1","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.442671971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"2","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.442693981Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"0","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443199457Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443299285Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[0]R flush_numbers[0..12]: [51, 58, 55, 47, 58, 59, 33, 41, 45, 45, 57, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443339313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[0]R generation[0..12]: [26, 28, 26, 25, 28, 28, 19, 23, 24, 24, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443356043Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443370813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[1]R flush_numbers[0..12]: [51, 58, 55, 47, 58, 59, 33, 41, 45, 45, 57, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443384942Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[1]R generation[0..12]: [26, 28, 26, 25, 28, 28, 19, 23, 24, 24, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443398572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443412471Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[2]R flush_numbers[0..12]: [51, 58, 55, 47, 58, 59, 33, 41, 45, 45, 57, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443425831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[2]R generation[0..12]: [26, 28, 26, 25, 28, 28, 19, 23, 24, 24, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443442841Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.44345689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.44346976Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.4434828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Next flush: 61","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-03-19T16:59:28.443508319Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443521669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"":"downstairs","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443534588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443546228Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"0","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443559048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"1","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443571717Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"client":"2","":"downstairs","session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443584467Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443596427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"f61c1587-44ed-4749-b2e7-c902adc498c7 is now active with session: f4c804e6-a639-425b-b00a-08df4938cdbb","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443608876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443620336Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312,"session_id":"f4c804e6-a639-425b-b00a-08df4938cdbb"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443632136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} {"msg":"Activated sub_volume f61c1587-44ed-4749-b2e7-c902adc498c7","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:28.443643036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 74 len 32768 data: 3 3 3 3 2 2 2 2 02/30 Read block 197 len 36864 03/30 Read block 537 len 32768 04/30 Read block 149 len 4096 05/30 Read block 161 len 32768 06/30 Write block 201 len 40960 data: 5 5 4 4 3 3 3 2 3 4 07/30 Read block 33 len 40960 08/30 Flush 09/30 Write block 564 len 40960 data: 3 3 3 2 3 3 3 3 3 2 10/30 Flush 11/30 Write block 548 len 32768 data: 6 5 6 6 5 6 5 5 12/30 Write block 331 len 36864 data: 5 5 5 5 5 5 5 6 4 13/30 Write block 572 len 28672 data: 4 3 2 3 3 3 3 14/30 Read block 342 len 12288 15/30 Write block 391 len 32768 data: 5 5 5 6 5 4 4 5 16/30 Read block 400 len 16384 17/30 Write block 400 len 16384 data: 4 4 4 3 18/30 Read block 568 len 16384 19/30 Read block 347 len 4096 20/30 Write block 372 len 12288 data: 3 3 3 21/30 Read block 338 len 24576 22/30 Write block 351 len 4096 data: 4 23/30 Flush 24/30 Write block 174 len 32768 data: 4 4 4 3 4 4 3 2 25/30 Read block 411 len 8192 26/30 Write block 369 len 8192 data: 2 3 27/30 Write block 562 len 36864 data: 3 3 4 4 4 3 4 4 4 28/30 Read block 126 len 28672 29/30 Read block 217 len 12288 30/30 Read block 222 len 12288 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:28.465653676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 28 28 28 58 58 58 F F F 2 040-059 26 26 26 55 55 55 F F F 3 060-079 30 30 30 61 61 61 F F F 4 080-099 30 30 30 61 61 61 F F F 5 100-119 28 28 28 59 59 59 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 24 24 24 45 45 45 T T T 9 180-199 24 24 24 45 45 45 T T T 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 29 29 29 60 60 60 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 25 25 25 47 47 47 F F F 16 320-339 30 30 30 63 63 63 F F F 17 340-359 30 30 30 63 63 63 F F F 18 360-379 30 30 30 63 63 63 T T T 19 380-399 30 30 30 63 63 63 F F F 20 400-419 30 30 30 63 63 63 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 29 29 29 60 60 60 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 28 28 28 58 58 58 F F F 26 520-539 27 27 27 56 56 56 F F F 27 540-559 30 30 30 63 63 63 F F F 28 560-579 30 30 30 63 63 63 T T T 29 580-599 28 28 28 57 57 57 F F F Max gen: 30, Max flush: 63 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 28 28 28 58 58 58 F F F 2 040-059 26 26 26 55 55 55 F F F 3 060-079 30 30 30 61 61 61 F F F 4 080-099 30 30 30 61 61 61 F F F 5 100-119 28 28 28 59 59 59 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 31 31 31 64 64 64 F F F 9 180-199 31 31 31 64 64 64 F F F 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 29 29 29 60 60 60 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 25 25 25 47 47 47 F F F 16 320-339 30 30 30 63 63 63 F F F 17 340-359 30 30 30 63 63 63 F F F 18 360-379 31 31 31 64 64 64 F F F 19 380-399 30 30 30 63 63 63 F F F 20 400-419 30 30 30 63 63 63 F F F 21 420-439 28 28 28 57 57 57 F F F 22 440-459 29 29 29 60 60 60 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 28 28 28 58 58 58 F F F 26 520-539 27 27 27 56 56 56 F F F 27 540-559 30 30 30 63 63 63 F F F 28 560-579 31 31 31 64 64 64 F F F 29 580-599 28 28 28 57 57 57 F F F Max gen: 31, Max flush: 64 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.728019915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.77300238Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.77377007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.773805249Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.773819148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.773833198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} {"msg":"Crucible aefc4f4e-a656-4001-9a7e-b919046f0e73 has session id: c3682b4b-0ce3-48ec-8bee-4341fecd64ca","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.774399743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Upstairs opts: Upstairs UUID: aefc4f4e-a656-4001-9a7e-b919046f0e73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.774445672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Crucible stats registered with UUID: aefc4f4e-a656-4001-9a7e-b919046f0e73","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.774470641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:32.774492221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.774927469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.774959838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} {"msg":"aefc4f4e-a656-4001-9a7e-b919046f0e73 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775017257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775043656Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"0","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775068075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"0","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775082285Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"1","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775106634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"1","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775131684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"2","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775158573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"2","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775196892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.77526266Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775300989Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:34590)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775331748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:37557)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775362878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:54967)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.775391077Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.77602449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"0","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.776068819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"2","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.776475858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"1","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.776739201Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"0","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.776773181Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"2","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777115171Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"1","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777359914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777471721Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[0]R flush_numbers[0..12]: [51, 58, 55, 61, 61, 59, 33, 41, 64, 64, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777670996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[0]R generation[0..12]: [26, 28, 26, 30, 30, 28, 19, 23, 31, 31, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777691535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777706465Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[1]R flush_numbers[0..12]: [51, 58, 55, 61, 61, 59, 33, 41, 64, 64, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777724004Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[1]R generation[0..12]: [26, 28, 26, 30, 30, 28, 19, 23, 31, 31, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777737774Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777751444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[2]R flush_numbers[0..12]: [51, 58, 55, 61, 61, 59, 33, 41, 64, 64, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777764874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[2]R generation[0..12]: [26, 28, 26, 30, 30, 28, 19, 23, 31, 31, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777778073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777791863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777804693Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777817522Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777830092Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777842781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"":"downstairs","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777855521Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777867111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"0","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.77787996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"1","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.77789273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"client":"2","":"downstairs","session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:59:32.77790712Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777931299Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"aefc4f4e-a656-4001-9a7e-b919046f0e73 is now active with session: c3682b4b-0ce3-48ec-8bee-4341fecd64ca","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777945289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777957168Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320,"session_id":"c3682b4b-0ce3-48ec-8bee-4341fecd64ca"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777969038Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} {"msg":"Activated sub_volume aefc4f4e-a656-4001-9a7e-b919046f0e73","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:32.777980028Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 53 len 8192 data: 5 5 02/30 Flush 03/30 Write block 366 len 12288 data: 3 3 3 04/30 Write block 86 len 28672 data: 4 4 4 6 3 4 3 05/30 Read block 95 len 20480 06/30 Read block 577 len 8192 07/30 Write block 75 len 36864 data: 4 4 4 3 3 3 3 2 3 08/30 Flush 09/30 Write block 431 len 28672 data: 5 4 3 4 4 3 3 10/30 Read block 138 len 8192 11/30 Flush 12/30 Write block 83 len 12288 data: 4 3 4 13/30 Read block 275 len 16384 14/30 Write block 300 len 4096 data: 3 15/30 Read block 569 len 40960 16/30 Read block 494 len 20480 17/30 Read block 322 len 8192 18/30 Read block 457 len 40960 19/30 Write block 361 len 20480 data: 4 6 5 4 3 20/30 Write block 442 len 28672 data: 2 2 2 3 4 3 3 21/30 Read block 38 len 12288 22/30 Write block 532 len 32768 data: 3 2 2 2 2 2 2 3 23/30 Read block 441 len 16384 24/30 Read block 73 len 8192 25/30 Write block 23 len 40960 data: 3 3 4 4 4 6 6 5 4 5 26/30 Read block 547 len 8192 27/30 Write block 381 len 4096 data: 5 28/30 Read block 182 len 4096 29/30 Read block 81 len 4096 30/30 Write block 508 len 8192 data: 3 4 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:32.797727837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 28 28 28 58 58 58 T T T 2 040-059 32 32 32 65 65 65 F F F 3 060-079 32 32 32 66 66 66 F F F 4 080-099 32 32 32 66 66 66 T T T 5 100-119 28 28 28 59 59 59 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 31 31 31 64 64 64 F F F 9 180-199 31 31 31 64 64 64 F F F 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 29 29 29 60 60 60 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 25 25 25 47 47 47 T T T 16 320-339 30 30 30 63 63 63 F F F 17 340-359 30 30 30 63 63 63 F F F 18 360-379 32 32 32 66 66 66 T T T 19 380-399 30 30 30 63 63 63 T T T 20 400-419 30 30 30 63 63 63 F F F 21 420-439 32 32 32 67 67 67 F F F 22 440-459 29 29 29 60 60 60 T T T 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 28 28 28 58 58 58 T T T 26 520-539 27 27 27 56 56 56 T T T 27 540-559 30 30 30 63 63 63 F F F 28 560-579 31 31 31 64 64 64 F F F 29 580-599 28 28 28 57 57 57 F F F Max gen: 32, Max flush: 67 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 33 33 33 68 68 68 F F F 2 040-059 32 32 32 65 65 65 F F F 3 060-079 32 32 32 66 66 66 F F F 4 080-099 33 33 33 68 68 68 F F F 5 100-119 28 28 28 59 59 59 F F F 6 120-139 19 19 19 33 33 33 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 31 31 31 64 64 64 F F F 9 180-199 31 31 31 64 64 64 F F F 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 29 29 29 60 60 60 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 33 33 33 68 68 68 F F F 16 320-339 30 30 30 63 63 63 F F F 17 340-359 30 30 30 63 63 63 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 33 33 33 68 68 68 F F F 20 400-419 30 30 30 63 63 63 F F F 21 420-439 32 32 32 67 67 67 F F F 22 440-459 33 33 33 68 68 68 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 26 26 26 51 51 51 F F F 25 500-519 33 33 33 68 68 68 F F F 26 520-539 33 33 33 68 68 68 F F F 27 540-559 30 30 30 63 63 63 F F F 28 560-579 31 31 31 64 64 64 F F F 29 580-599 28 28 28 57 57 57 F F F Max gen: 33, Max flush: 68 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.049267582Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:37.091059681Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.091446221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.122263989Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.122885843Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.122915492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.122928311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.122942581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} {"msg":"Crucible 6043a7b3-6170-4d25-a8a7-1aaeee18db61 has session id: d73985f2-5702-423e-8ce5-8c79dd7d05ff","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.123621303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Upstairs opts: Upstairs UUID: 6043a7b3-6170-4d25-a8a7-1aaeee18db61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.123667752Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Crucible stats registered with UUID: 6043a7b3-6170-4d25-a8a7-1aaeee18db61","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.123695961Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:37.123714151Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.12410719Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.12413026Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} {"msg":"6043a7b3-6170-4d25-a8a7-1aaeee18db61 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124174709Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124190718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"0","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124214917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"0","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124229697Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"1","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124244807Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"1","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124267946Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"2","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124294825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"2","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124320725Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124346674Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124370373Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:60922)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124397253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:41341)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124424392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:36084)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.124596198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.125051615Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"2","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.125168252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"1","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.12525761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"0","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.125761977Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"2","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.125783186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"1","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.125955192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"0","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.12639427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126429929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[0]R flush_numbers[0..12]: [51, 68, 65, 66, 68, 59, 33, 41, 64, 64, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126501857Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[0]R generation[0..12]: [26, 33, 32, 32, 33, 28, 19, 23, 31, 31, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126529916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126545176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[1]R flush_numbers[0..12]: [51, 68, 65, 66, 68, 59, 33, 41, 64, 64, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126558746Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[1]R generation[0..12]: [26, 33, 32, 32, 33, 28, 19, 23, 31, 31, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126572255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126586265Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[2]R flush_numbers[0..12]: [51, 68, 65, 66, 68, 59, 33, 41, 64, 64, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126599635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[2]R generation[0..12]: [26, 33, 32, 32, 33, 28, 19, 23, 31, 31, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126612834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126629664Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126642453Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126655123Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126667533Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126680203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"":"downstairs","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126693022Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T16:59:37.126704622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"0","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126723602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"1","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126736331Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"client":"2","":"downstairs","session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126749071Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.12676089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"6043a7b3-6170-4d25-a8a7-1aaeee18db61 is now active with session: d73985f2-5702-423e-8ce5-8c79dd7d05ff","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.12677353Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.1267851Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328,"session_id":"d73985f2-5702-423e-8ce5-8c79dd7d05ff"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126802619Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} {"msg":"Activated sub_volume 6043a7b3-6170-4d25-a8a7-1aaeee18db61","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:37.126813869Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 36864 data: 3 3 2 2 2 3 4 3 3 02/30 Write block 438 len 20480 data: 2 2 2 3 3 03/30 Read block 96 len 20480 04/30 Read block 138 len 4096 05/30 Write block 399 len 12288 data: 4 5 5 06/30 Read block 332 len 12288 07/30 Write block 164 len 40960 data: 5 5 5 5 5 4 5 5 5 5 08/30 Read block 407 len 36864 09/30 Read block 516 len 12288 10/30 Write block 422 len 20480 data: 4 5 5 4 3 11/30 Read block 338 len 28672 12/30 Write block 495 len 16384 data: 4 4 3 3 13/30 Write block 547 len 20480 data: 7 7 6 7 7 14/30 Read block 515 len 40960 15/30 Flush 16/30 Write block 183 len 8192 data: 3 3 17/30 Write block 114 len 4096 data: 3 18/30 Read block 594 len 8192 19/30 Write block 398 len 4096 data: 6 20/30 Write block 300 len 40960 data: 4 2 3 3 3 2 3 3 3 4 21/30 Read block 335 len 16384 22/30 Read block 431 len 36864 23/30 Write block 50 len 16384 data: 5 4 4 6 24/30 Read block 409 len 12288 25/30 Read block 17 len 40960 26/30 Write block 546 len 8192 data: 6 8 27/30 Write block 273 len 8192 data: 4 4 28/30 Read block 545 len 36864 29/30 Write block 262 len 8192 data: 5 3 30/30 Read block 440 len 4096 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 2 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 13 15 13 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:37.146977478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 33 33 33 68 68 68 F F F 2 040-059 32 32 32 65 65 65 T T T 3 060-079 32 32 32 66 66 66 F F F 4 080-099 33 33 33 68 68 68 F F F 5 100-119 34 34 34 69 69 69 T T T 6 120-139 34 34 34 69 69 69 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 34 34 34 69 69 69 F F F 9 180-199 31 31 31 64 64 64 T T T 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 29 29 29 60 60 60 T T T 14 280-299 28 28 28 58 58 58 F F F 15 300-319 33 33 33 68 68 68 T T T 16 320-339 30 30 30 63 63 63 F F F 17 340-359 30 30 30 63 63 63 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 34 34 34 69 69 69 T T T 20 400-419 34 34 34 69 69 69 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 33 33 33 68 68 68 F F F 26 520-539 33 33 33 68 68 68 F F F 27 540-559 34 34 34 69 69 69 T T T 28 560-579 31 31 31 64 64 64 F F F 29 580-599 28 28 28 57 57 57 F F F Max gen: 34, Max flush: 69 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 33 33 33 68 68 68 F F F 2 040-059 35 35 35 70 70 70 F F F 3 060-079 32 32 32 66 66 66 F F F 4 080-099 33 33 33 68 68 68 F F F 5 100-119 35 35 35 70 70 70 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 23 23 23 41 41 41 F F F 8 160-179 34 34 34 69 69 69 F F F 9 180-199 35 35 35 70 70 70 F F F 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 28 28 28 58 58 58 F F F 13 260-279 35 35 35 70 70 70 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 35 35 35 70 70 70 F F F 16 320-339 30 30 30 63 63 63 F F F 17 340-359 30 30 30 63 63 63 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 34 34 34 69 69 69 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 19 19 19 33 33 33 F F F 24 480-499 34 34 34 69 69 69 F F F 25 500-519 33 33 33 68 68 68 F F F 26 520-539 33 33 33 68 68 68 F F F 27 540-559 35 35 35 70 70 70 F F F 28 560-579 31 31 31 64 64 64 F F F 29 580-599 28 28 28 57 57 57 F F F Max gen: 35, Max flush: 70 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.423192281Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.465880167Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.46652774Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.466562029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.466575808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.466590018Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} {"msg":"Crucible 7bfb655c-4325-46fa-97b0-293119747d66 has session id: 940b03d2-92ff-4a2b-b3d7-194d7bd17a6c","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467197022Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Upstairs opts: Upstairs UUID: 7bfb655c-4325-46fa-97b0-293119747d66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467231371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Crucible stats registered with UUID: 7bfb655c-4325-46fa-97b0-293119747d66","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467246541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:41.46725914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467685069Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467704029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} {"msg":"7bfb655c-4325-46fa-97b0-293119747d66 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467772617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467791186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"0","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467816556Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"0","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467842265Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467868194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467894694Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"2","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467926303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"2","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467954002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.467985211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.468012431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56275)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.46803997Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65076)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.468068529Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:41.468218585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:41.468253724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:41.468274984Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.468464089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.468882877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"0","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.468916557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"2","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.469446913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"2","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.46954698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"0","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.470019138Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:41.470099825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.468899981Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:32834)","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.468966879Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.470034891Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.470790031Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[0]R flush_numbers[0..12]: [51, 68, 70, 66, 68, 70, 69, 41, 69, 70, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471425635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[0]R generation[0..12]: [26, 33, 35, 32, 33, 35, 34, 23, 34, 35, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471454134Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471471033Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[1]R flush_numbers[0..12]: [51, 68, 70, 66, 68, 70, 69, 41, 69, 70, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471484943Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[1]R generation[0..12]: [26, 33, 35, 32, 33, 35, 34, 23, 34, 35, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471498663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471512532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[2]R flush_numbers[0..12]: [51, 68, 70, 66, 68, 70, 69, 41, 69, 70, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471525532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[2]R generation[0..12]: [26, 33, 35, 32, 33, 35, 34, 23, 34, 35, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471538772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471552371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471565001Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.47158214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.47159535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.47160812Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"":"downstairs","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471621139Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471632769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"0","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471645799Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"1","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471658529Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"client":"2","":"downstairs","session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471671258Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471682978Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"7bfb655c-4325-46fa-97b0-293119747d66 is now active with session: 940b03d2-92ff-4a2b-b3d7-194d7bd17a6c","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471695468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471707157Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336,"session_id":"940b03d2-92ff-4a2b-b3d7-194d7bd17a6c"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471719007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} {"msg":"Activated sub_volume 7bfb655c-4325-46fa-97b0-293119747d66","v":0,"name":"crucible","level":30,"time":"2025-03-19T16:59:51.471729756Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 479 len 12288 data: 4 6 5 02/30 Write block 534 len 24576 data: 3 3 3 3 3 4 03/30 Write block 21 len 40960 data: 4 4 4 4 5 5 5 7 7 6 04/30 Write block 86 len 20480 data: 5 5 5 7 4 05/30 Write block 493 len 16384 data: 3 4 5 5 06/30 Read block 445 len 8192 07/30 Write block 249 len 16384 data: 2 4 2 3 08/30 Write block 241 len 12288 data: 4 5 6 09/30 Read block 0 len 12288 10/30 Flush 11/30 Write block 587 len 4096 data: 4 12/30 Flush 13/30 Write block 338 len 36864 data: 7 5 5 4 4 5 4 4 3 14/30 Flush 15/30 Write block 93 len 36864 data: 4 3 3 5 5 5 5 6 6 16/30 Read block 545 len 28672 17/30 Read block 121 len 12288 18/30 Read block 12 len 20480 19/30 Write block 531 len 40960 data: 3 4 3 4 4 4 4 4 5 3 20/30 Read block 501 len 20480 21/30 Write block 156 len 8192 data: 3 3 22/30 Read block 190 len 12288 23/30 Read block 67 len 16384 24/30 Write block 264 len 20480 data: 3 3 3 3 4 25/30 Write block 567 len 8192 data: 4 5 26/30 Read block 178 len 12288 27/30 Write block 483 len 12288 data: 3 3 3 28/30 Read block 277 len 32768 29/30 Read block 391 len 16384 30/30 Read block 25 len 40960 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T16:59:51.492177878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 36 36 36 71 71 71 F F F 2 040-059 35 35 35 70 70 70 F F F 3 060-079 32 32 32 66 66 66 F F F 4 080-099 36 36 36 71 71 71 T T T 5 100-119 35 35 35 70 70 70 T T T 6 120-139 34 34 34 69 69 69 F F F 7 140-159 23 23 23 41 41 41 T T T 8 160-179 34 34 34 69 69 69 F F F 9 180-199 35 35 35 70 70 70 F F F 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 35 35 35 70 70 70 T T T 14 280-299 28 28 28 58 58 58 F F F 15 300-319 35 35 35 70 70 70 F F F 16 320-339 36 36 36 73 73 73 F F F 17 340-359 36 36 36 73 73 73 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 34 34 34 69 69 69 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 36 36 36 71 71 71 F F F 24 480-499 36 36 36 71 71 71 T T T 25 500-519 33 33 33 68 68 68 F F F 26 520-539 36 36 36 71 71 71 T T T 27 540-559 35 35 35 70 70 70 T T T 28 560-579 31 31 31 64 64 64 T T T 29 580-599 36 36 36 72 72 72 F F F Max gen: 36, Max flush: 73 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 36 36 36 71 71 71 F F F 2 040-059 35 35 35 70 70 70 F F F 3 060-079 32 32 32 66 66 66 F F F 4 080-099 37 37 37 74 74 74 F F F 5 100-119 37 37 37 74 74 74 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 37 37 37 74 74 74 F F F 8 160-179 34 34 34 69 69 69 F F F 9 180-199 35 35 35 70 70 70 F F F 10 200-219 30 30 30 61 61 61 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 37 37 37 74 74 74 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 35 35 35 70 70 70 F F F 16 320-339 36 36 36 73 73 73 F F F 17 340-359 36 36 36 73 73 73 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 34 34 34 69 69 69 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 36 36 36 71 71 71 F F F 24 480-499 37 37 37 74 74 74 F F F 25 500-519 33 33 33 68 68 68 F F F 26 520-539 37 37 37 74 74 74 F F F 27 540-559 37 37 37 74 74 74 F F F 28 560-579 37 37 37 74 74 74 F F F 29 580-599 36 36 36 72 72 72 F F F Max gen: 37, Max flush: 74 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.754583198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.80228584Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.803332053Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.803371992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.803386631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.803401191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344} {"msg":"Crucible 9e339e75-0f9e-4c79-8d5e-b38012d2e766 has session id: 49181579-f08f-4102-a891-001973c5828e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804017165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Upstairs opts: Upstairs UUID: 9e339e75-0f9e-4c79-8d5e-b38012d2e766, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804095073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Crucible stats registered with UUID: 9e339e75-0f9e-4c79-8d5e-b38012d2e766","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804119752Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:05.804142811Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.80458525Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804613939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344} {"msg":"9e339e75-0f9e-4c79-8d5e-b38012d2e766 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804677527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804694677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"0","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804719566Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"0","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804733996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"1","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804748235Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"1","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804761235Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"2","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804775465Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"2","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804793114Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804857463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804888152Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52436)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804915511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56921)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.80494922Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62811)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.804980509Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.805811787Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"0","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.805839057Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"2","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.806306234Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"1","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.806503969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"0","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.806529508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"2","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.806972257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"1","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807175402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807199011Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[0]R flush_numbers[0..12]: [51, 71, 70, 66, 74, 74, 69, 74, 69, 70, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.80760859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[0]R generation[0..12]: [26, 36, 35, 32, 37, 37, 34, 37, 34, 35, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807628019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807642129Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[1]R flush_numbers[0..12]: [51, 71, 70, 66, 74, 74, 69, 74, 69, 70, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807655389Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[1]R generation[0..12]: [26, 36, 35, 32, 37, 37, 34, 37, 34, 35, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807672558Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807686628Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[2]R flush_numbers[0..12]: [51, 71, 70, 66, 74, 74, 69, 74, 69, 70, 61, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807699848Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[2]R generation[0..12]: [26, 36, 35, 32, 37, 37, 34, 37, 34, 35, 30, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807712957Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807726887Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807739367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807752186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807764686Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807777276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"":"downstairs","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807790135Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807801835Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"0","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807814795Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"1","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807827494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"client":"2","":"downstairs","session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807840184Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807852224Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"9e339e75-0f9e-4c79-8d5e-b38012d2e766 is now active with session: 49181579-f08f-4102-a891-001973c5828e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807866893Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807878773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344,"session_id":"49181579-f08f-4102-a891-001973c5828e"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807890673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344Wait for a query_work_queue command to finish before sending IO } {"msg":"Activated sub_volume 9e339e75-0f9e-4c79-8d5e-b38012d2e766","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:05.807910492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 36864 data: 3 2 2 4 4 4 3 3 3 02/30 Read block 157 len 28672 03/30 Read block 169 len 36864 04/30 Read block 1 len 16384 05/30 Write block 537 len 16384 data: 5 5 6 4 06/30 Read block 134 len 8192 07/30 Read block 123 len 32768 08/30 Read block 185 len 40960 09/30 Write block 104 len 24576 data: 5 5 3 3 3 3 10/30 Read block 544 len 28672 11/30 Write block 198 len 32768 data: 3 4 5 6 6 5 5 4 12/30 Write block 343 len 20480 data: 6 5 5 4 3 13/30 Read block 405 len 24576 14/30 Read block 504 len 8192 15/30 Read block 324 len 20480 16/30 Read block 518 len 12288 17/30 Read block 297 len 28672 18/30 Read block 495 len 32768 19/30 Read block 438 len 24576 20/30 Flush 21/30 Write block 507 len 16384 data: 2 4 5 4 22/30 Read block 114 len 20480 23/30 Write block 293 len 12288 data: 4 5 4 24/30 Write block 24 len 32768 data: 5 6 6 6 8 8 7 5 25/30 Read block 188 len 36864 26/30 Write block 75 len 40960 data: 5 5 5 4 4 4 4 3 5 4 27/30 Write block 309 len 32768 data: 5 4 4 5 5 5 4 3 28/30 Read block 585 len 28672 29/30 Read block 140 len 4096 30/30 Write block 40 len 24576 data: 5 5 5 5 5 5 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:05.831217908Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 36 36 36 71 71 71 T T T 2 040-059 35 35 35 70 70 70 T T T 3 060-079 32 32 32 66 66 66 T T T 4 080-099 37 37 37 74 74 74 T T T 5 100-119 38 38 38 75 75 75 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 38 38 38 75 75 75 F F F 10 200-219 38 38 38 75 75 75 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 37 37 37 74 74 74 F F F 14 280-299 28 28 28 58 58 58 T T T 15 300-319 35 35 35 70 70 70 T T T 16 320-339 36 36 36 73 73 73 F F F 17 340-359 38 38 38 75 75 75 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 34 34 34 69 69 69 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 36 36 36 71 71 71 F F F 24 480-499 37 37 37 74 74 74 F F F 25 500-519 33 33 33 68 68 68 T T T 26 520-539 38 38 38 75 75 75 F F F 27 540-559 38 38 38 75 75 75 F F F 28 560-579 37 37 37 74 74 74 F F F 29 580-599 36 36 36 72 72 72 F F F Max gen: 38, Max flush: 75 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 39 39 39 76 76 76 F F F 2 040-059 39 39 39 76 76 76 F F F 3 060-079 39 39 39 76 76 76 F F F 4 080-099 39 39 39 76 76 76 F F F 5 100-119 38 38 38 75 75 75 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 38 38 38 75 75 75 F F F 10 200-219 38 38 38 75 75 75 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 37 37 37 74 74 74 F F F 14 280-299 39 39 39 76 76 76 F F F 15 300-319 39 39 39 76 76 76 F F F 16 320-339 36 36 36 73 73 73 F F F 17 340-359 38 38 38 75 75 75 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 34 34 34 69 69 69 F F F 21 420-439 34 34 34 69 69 69 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 36 36 36 71 71 71 F F F 24 480-499 37 37 37 74 74 74 F F F 25 500-519 39 39 39 76 76 76 F F F 26 520-539 38 38 38 75 75 75 F F F 27 540-559 38 38 38 75 75 75 F F F 28 560-579 37 37 37 74 74 74 F F F 29 580-599 36 36 36 72 72 72 F F F Max gen: 39, Max flush: 76 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.084766142Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.128415372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.129132283Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.129166562Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.129184772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.129199251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} {"msg":"Crucible f6d94bb8-d1e7-452b-866f-15959b91134c has session id: 73440784-f959-4763-a00a-6fa64ef978c2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.129819765Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Upstairs opts: Upstairs UUID: f6d94bb8-d1e7-452b-866f-15959b91134c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.129857524Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Crucible stats registered with UUID: f6d94bb8-d1e7-452b-866f-15959b91134c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.129873114Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:20.129885813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130311642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130336201Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} {"msg":"f6d94bb8-d1e7-452b-866f-15959b91134c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.13038854Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130414639Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"0","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130439109Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"0","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130453538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130468418Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130494677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"2","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130536156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"2","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130562865Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130587675Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130612534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36114)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130636833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38091)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.130660793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:20.130846218Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:20.130891577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:20.130918986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.131011644Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.131409493Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"0","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.131449142Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"2","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.131958068Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"2","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.132060296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"0","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.132483515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:20.132597502Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.132119125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43091)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.132227892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.133218286Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.133923168Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[0]R flush_numbers[0..12]: [51, 76, 76, 76, 76, 75, 69, 75, 75, 75, 75, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134537972Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[0]R generation[0..12]: [26, 39, 39, 39, 39, 38, 34, 38, 38, 38, 38, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134566601Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.13458297Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[1]R flush_numbers[0..12]: [51, 76, 76, 76, 76, 75, 69, 75, 75, 75, 75, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.13459698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[1]R generation[0..12]: [26, 39, 39, 39, 39, 38, 34, 38, 38, 38, 38, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.13461017Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134624079Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[2]R flush_numbers[0..12]: [51, 76, 76, 76, 76, 75, 69, 75, 75, 75, 75, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134637729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[2]R generation[0..12]: [26, 39, 39, 39, 39, 38, 34, 38, 38, 38, 38, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134655159Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134669938Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134682678Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134695407Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134707827Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134720277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"":"downstairs","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134733417Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134745076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"0","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134758236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"1","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134771196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"client":"2","":"downstairs","session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:00:30.134784275Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134800915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"f6d94bb8-d1e7-452b-866f-15959b91134c is now active with session: 73440784-f959-4763-a00a-6fa64ef978c2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134813334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134824944Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352,"session_id":"73440784-f959-4763-a00a-6fa64ef978c2"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134836854Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} {"msg":"Activated sub_volume f6d94bb8-d1e7-452b-866f-15959b91134c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:30.134847703Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 527 len 40960 data: 3 3 3 3 4 5 4 5 5 5 02/30 Read block 580 len 16384 03/30 Write block 50 len 12288 data: 6 5 5 04/30 Write block 314 len 12288 data: 6 5 4 05/30 Write block 281 len 28672 data: 3 3 3 3 3 2 2 06/30 Write block 403 len 36864 data: 4 3 3 2 2 2 2 3 3 07/30 Write block 93 len 32768 data: 5 4 4 6 6 6 6 7 08/30 Read block 332 len 32768 09/30 Read block 49 len 12288 10/30 Read block 199 len 20480 11/30 Flush 12/30 Write block 563 len 32768 data: 4 5 5 5 5 6 5 5 13/30 Write block 552 len 28672 data: 6 7 6 6 4 4 3 14/30 Write block 547 len 40960 data: 9 8 7 8 8 7 8 7 7 5 15/30 Read block 162 len 8192 16/30 Flush 17/30 Write block 507 len 8192 data: 3 5 18/30 Read block 454 len 8192 19/30 Read block 238 len 20480 20/30 Write block 333 len 40960 data: 6 6 6 6 6 8 6 6 5 5 21/30 Read block 427 len 24576 22/30 Flush 23/30 Write block 422 len 40960 data: 5 6 6 5 4 3 4 3 4 6 24/30 Read block 313 len 32768 25/30 Write block 214 len 24576 data: 3 3 3 2 2 2 26/30 Write block 566 len 4096 data: 6 27/30 Write block 305 len 28672 data: 3 4 4 4 6 5 5 28/30 Read block 451 len 20480 29/30 Write block 578 len 40960 data: 4 3 3 3 3 3 2 2 2 5 30/30 Read block 99 len 4096 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:30.156845474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 39 39 39 76 76 76 F F F 2 040-059 40 40 40 77 77 77 F F F 3 060-079 39 39 39 76 76 76 F F F 4 080-099 40 40 40 77 77 77 F F F 5 100-119 40 40 40 77 77 77 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 38 38 38 75 75 75 F F F 10 200-219 38 38 38 75 75 75 T T T 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 37 37 37 74 74 74 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 40 40 40 77 77 77 T T T 16 320-339 40 40 40 79 79 79 F F F 17 340-359 40 40 40 79 79 79 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 40 40 40 77 77 77 F F F 21 420-439 34 34 34 69 69 69 T T T 22 440-459 34 34 34 69 69 69 F F F 23 460-479 36 36 36 71 71 71 F F F 24 480-499 37 37 37 74 74 74 F F F 25 500-519 40 40 40 79 79 79 F F F 26 520-539 40 40 40 77 77 77 F F F 27 540-559 40 40 40 78 78 78 F F F 28 560-579 40 40 40 78 78 78 T T T 29 580-599 36 36 36 72 72 72 T T T Max gen: 40, Max flush: 79 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 39 39 39 76 76 76 F F F 2 040-059 40 40 40 77 77 77 F F F 3 060-079 39 39 39 76 76 76 F F F 4 080-099 40 40 40 77 77 77 F F F 5 100-119 40 40 40 77 77 77 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 38 38 38 75 75 75 F F F 10 200-219 41 41 41 80 80 80 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 37 37 37 74 74 74 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 41 41 41 80 80 80 F F F 16 320-339 40 40 40 79 79 79 F F F 17 340-359 40 40 40 79 79 79 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 40 40 40 77 77 77 F F F 21 420-439 41 41 41 80 80 80 F F F 22 440-459 34 34 34 69 69 69 F F F 23 460-479 36 36 36 71 71 71 F F F 24 480-499 37 37 37 74 74 74 F F F 25 500-519 40 40 40 79 79 79 F F F 26 520-539 40 40 40 77 77 77 F F F 27 540-559 40 40 40 78 78 78 F F F 28 560-579 41 41 41 80 80 80 F F F 29 580-599 41 41 41 80 80 80 F F F Max gen: 41, Max flush: 80 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.399693578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:34.441993384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.442440942Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.473495884Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.474225705Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.474264114Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.474282833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.474297923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} {"msg":"Crucible b705cd63-86e0-40b8-a3a8-864dbc492890 has session id: 480b3364-99e0-4c52-a292-a59a30b05496","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.474866568Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Upstairs opts: Upstairs UUID: b705cd63-86e0-40b8-a3a8-864dbc492890, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.474914266Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Crucible stats registered with UUID: b705cd63-86e0-40b8-a3a8-864dbc492890","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.474941126Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:34.474955875Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475299606Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475321196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} {"msg":"b705cd63-86e0-40b8-a3a8-864dbc492890 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475388634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475405944Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"0","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475429643Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"0","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475443343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"1","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475457192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"1","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475470622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"2","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475495021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"2","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.47552717Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475600298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475620318Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"ds_connection connected from Ok(127.0.0.1:41396)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475645187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"ds_connection connected from Ok(127.0.0.1:38471)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475670186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"ds_connection connected from Ok(127.0.0.1:33679)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.475695336Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.476426267Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"1","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.476455636Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"0","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.476483845Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"2","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477142338Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"1","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477185377Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"2","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477215806Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"0","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477715633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477742242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[0]R flush_numbers[0..12]: [51, 76, 77, 76, 77, 77, 69, 75, 75, 75, 80, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.47779853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[0]R generation[0..12]: [26, 39, 40, 39, 40, 40, 34, 38, 38, 38, 41, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477834229Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477850149Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[1]R flush_numbers[0..12]: [51, 76, 77, 76, 77, 77, 69, 75, 75, 75, 80, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477864069Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[1]R generation[0..12]: [26, 39, 40, 39, 40, 40, 34, 38, 38, 38, 41, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477877498Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477891268Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[2]R flush_numbers[0..12]: [51, 76, 77, 76, 77, 77, 69, 75, 75, 75, 80, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477904828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[2]R generation[0..12]: [26, 39, 40, 39, 40, 40, 34, 38, 38, 38, 41, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477917877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477931597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477944387Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477957646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477970266Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.477982996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"":"downstairs","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:00:34.478018045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478039034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"0","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478053624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"1","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478066894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"client":"2","":"downstairs","session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478079863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478091843Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"b705cd63-86e0-40b8-a3a8-864dbc492890 is now active with session: 480b3364-99e0-4c52-a292-a59a30b05496","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478104233Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478115932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360,"session_id":"480b3364-99e0-4c52-a292-a59a30b05496"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478127712Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} {"msg":"Activated sub_volume b705cd63-86e0-40b8-a3a8-864dbc492890","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:34.478138621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 419 len 28672 data: 8 8 6 6 7 7 6 02/30 Write block 454 len 24576 data: 5 5 4 3 4 3 03/30 Read block 493 len 12288 04/30 Read block 315 len 36864 05/30 Flush 06/30 Write block 217 len 8192 data: 3 3 07/30 Write block 406 len 32768 data: 3 3 3 3 4 4 5 5 08/30 Read block 204 len 24576 09/30 Write block 184 len 8192 data: 4 3 10/30 Read block 65 len 40960 11/30 Flush 12/30 Write block 542 len 32768 data: 5 8 8 8 7 10 9 8 13/30 Read block 534 len 20480 14/30 Read block 540 len 24576 15/30 Write block 188 len 28672 data: 3 3 4 4 4 2 2 16/30 Write block 265 len 24576 data: 4 4 4 5 4 4 17/30 Write block 189 len 36864 data: 4 5 5 5 3 3 3 4 3 18/30 Write block 90 len 36864 data: 5 5 4 6 5 5 7 7 7 19/30 Read block 191 len 16384 20/30 Read block 274 len 20480 21/30 Write block 99 len 16384 data: 7 8 7 6 22/30 Read block 514 len 24576 23/30 Write block 59 len 32768 data: 2 2 2 4 5 4 4 4 24/30 Write block 212 len 4096 data: 5 25/30 Write block 544 len 40960 data: 9 9 8 11 10 9 9 9 8 9 26/30 Write block 457 len 28672 data: 4 5 4 2 2 2 2 27/30 Read block 261 len 36864 28/30 Read block 21 len 12288 29/30 Read block 352 len 12288 30/30 Read block 139 len 36864 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 8 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:34.499184397Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 39 39 39 76 76 76 F F F 2 040-059 40 40 40 77 77 77 T T T 3 060-079 39 39 39 76 76 76 T T T 4 080-099 40 40 40 77 77 77 T T T 5 100-119 40 40 40 77 77 77 T T T 6 120-139 34 34 34 69 69 69 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 42 42 42 82 82 82 T T T 10 200-219 42 42 42 82 82 82 T T T 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 37 37 37 74 74 74 T T T 14 280-299 40 40 40 77 77 77 F F F 15 300-319 41 41 41 80 80 80 F F F 16 320-339 40 40 40 79 79 79 F F F 17 340-359 40 40 40 79 79 79 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 42 42 42 81 81 81 F F F 22 440-459 42 42 42 81 81 81 T T T 23 460-479 36 36 36 71 71 71 T T T 24 480-499 37 37 37 74 74 74 F F F 25 500-519 40 40 40 79 79 79 F F F 26 520-539 40 40 40 77 77 77 F F F 27 540-559 40 40 40 78 78 78 T T T 28 560-579 41 41 41 80 80 80 F F F 29 580-599 41 41 41 80 80 80 F F F Max gen: 42, Max flush: 82 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 39 39 39 76 76 76 F F F 2 040-059 43 43 43 83 83 83 F F F 3 060-079 43 43 43 83 83 83 F F F 4 080-099 43 43 43 83 83 83 F F F 5 100-119 43 43 43 83 83 83 F F F 6 120-139 34 34 34 69 69 69 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 43 43 43 83 83 83 F F F 10 200-219 43 43 43 83 83 83 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 36 36 36 71 71 71 F F F 13 260-279 43 43 43 83 83 83 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 41 41 41 80 80 80 F F F 16 320-339 40 40 40 79 79 79 F F F 17 340-359 40 40 40 79 79 79 F F F 18 360-379 33 33 33 68 68 68 F F F 19 380-399 35 35 35 70 70 70 F F F 20 400-419 42 42 42 82 82 82 F F F 21 420-439 42 42 42 81 81 81 F F F 22 440-459 43 43 43 83 83 83 F F F 23 460-479 43 43 43 83 83 83 F F F 24 480-499 37 37 37 74 74 74 F F F 25 500-519 40 40 40 79 79 79 F F F 26 520-539 40 40 40 77 77 77 F F F 27 540-559 43 43 43 83 83 83 F F F 28 560-579 41 41 41 80 80 80 F F F 29 580-599 41 41 41 80 80 80 F F F Max gen: 43, Max flush: 83 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.797290443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.842308377Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.843092366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.843127805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.843141515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.843155775Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} {"msg":"Crucible e42f6135-139e-4f8f-b04c-88533999a2d1 has session id: d99f02eb-92d6-4c58-8fe0-a50eb5642ffe","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.84371743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Upstairs opts: Upstairs UUID: e42f6135-139e-4f8f-b04c-88533999a2d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.843760329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Crucible stats registered with UUID: e42f6135-139e-4f8f-b04c-88533999a2d1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.843787548Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:38.843810127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844282235Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844318334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} {"msg":"e42f6135-139e-4f8f-b04c-88533999a2d1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844377223Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844417322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"0","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.84447908Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"0","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844519029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"1","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844546608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"1","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844608926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"2","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844645516Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"2","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844700704Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844726463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844752273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:56069)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844779972Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:44435)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844808741Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:39709)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.844892409Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.845521942Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"2","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.845566111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"0","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.845879983Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"1","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.846243183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"0","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.846282742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"2","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.846453588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"1","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.846894226Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.846919906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[0]R flush_numbers[0..12]: [51, 76, 83, 83, 83, 83, 69, 75, 75, 83, 83, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847037013Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[0]R generation[0..12]: [26, 39, 43, 43, 43, 43, 34, 38, 38, 43, 43, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847055932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847070392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[1]R flush_numbers[0..12]: [51, 76, 83, 83, 83, 83, 69, 75, 75, 83, 83, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847084991Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[1]R generation[0..12]: [26, 39, 43, 43, 43, 43, 34, 38, 38, 43, 43, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847098661Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847112281Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[2]R flush_numbers[0..12]: [51, 76, 83, 83, 83, 83, 69, 75, 75, 83, 83, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.8471256Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[2]R generation[0..12]: [26, 39, 43, 43, 43, 43, 34, 38, 38, 43, 43, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.84714822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847162899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847175689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847188478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:00:38.847201158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847221058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"":"downstairs","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847233947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847245707Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"0","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847258577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"1","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847271347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"client":"2","":"downstairs","session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847284086Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847295956Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"e42f6135-139e-4f8f-b04c-88533999a2d1 is now active with session: d99f02eb-92d6-4c58-8fe0-a50eb5642ffe","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847308446Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847320275Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368,"session_id":"d99f02eb-92d6-4c58-8fe0-a50eb5642ffe"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847332105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} {"msg":"Activated sub_volume e42f6135-139e-4f8f-b04c-88533999a2d1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:38.847344805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 40960 data: 9 9 8 6 6 5 6 6 6 6 02/30 Flush 03/30 Write block 247 len 24576 data: 5 4 3 5 3 4 04/30 Write block 379 len 36864 data: 4 5 6 5 5 5 4 4 4 05/30 Read block 60 len 8192 06/30 Flush 07/30 Write block 137 len 12288 data: 5 5 4 08/30 Read block 66 len 40960 09/30 Read block 49 len 28672 10/30 Write block 56 len 32768 data: 3 3 2 3 3 3 5 6 11/30 Write block 248 len 8192 data: 5 4 12/30 Write block 473 len 40960 data: 4 4 4 4 4 4 5 7 6 3 13/30 Read block 186 len 16384 14/30 Write block 88 len 40960 data: 6 8 6 6 5 7 6 6 8 8 15/30 Read block 427 len 36864 16/30 Write block 344 len 32768 data: 6 6 5 4 5 4 4 5 17/30 Write block 379 len 8192 data: 5 6 18/30 Write block 316 len 36864 data: 5 4 3 4 4 4 3 4 3 19/30 Flush 20/30 Write block 166 len 4096 data: 6 21/30 Read block 423 len 32768 22/30 Read block 449 len 40960 23/30 Write block 83 len 16384 data: 6 5 5 6 24/30 Write block 547 len 24576 data: 12 11 10 10 10 9 25/30 Read block 416 len 8192 26/30 Write block 442 len 28672 data: 4 3 3 4 5 4 4 27/30 Write block 398 len 16384 data: 7 5 6 6 28/30 Write block 247 len 20480 data: 6 6 5 6 4 29/30 Write block 198 len 32768 data: 4 5 6 7 7 6 6 5 30/30 Write block 564 len 20480 data: 6 6 7 6 7 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:38.867658509Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 44 44 44 84 84 84 F F F 2 040-059 44 44 44 86 86 86 F F F 3 060-079 44 44 44 86 86 86 F F F 4 080-099 44 44 44 86 86 86 T T T 5 100-119 43 43 43 83 83 83 F F F 6 120-139 44 44 44 86 86 86 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 T T T 9 180-199 43 43 43 83 83 83 T T T 10 200-219 43 43 43 83 83 83 T T T 11 220-239 27 27 27 56 56 56 F F F 12 240-259 44 44 44 86 86 86 T T T 13 260-279 43 43 43 83 83 83 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 44 44 44 86 86 86 F F F 16 320-339 44 44 44 86 86 86 F F F 17 340-359 44 44 44 86 86 86 F F F 18 360-379 44 44 44 86 86 86 F F F 19 380-399 44 44 44 86 86 86 T T T 20 400-419 42 42 42 82 82 82 T T T 21 420-439 42 42 42 81 81 81 F F F 22 440-459 43 43 43 83 83 83 T T T 23 460-479 44 44 44 86 86 86 F F F 24 480-499 44 44 44 86 86 86 F F F 25 500-519 40 40 40 79 79 79 F F F 26 520-539 40 40 40 77 77 77 F F F 27 540-559 43 43 43 83 83 83 T T T 28 560-579 41 41 41 80 80 80 T T T 29 580-599 41 41 41 80 80 80 F F F Max gen: 44, Max flush: 86 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 F F F 1 020-039 44 44 44 84 84 84 F F F 2 040-059 44 44 44 86 86 86 F F F 3 060-079 44 44 44 86 86 86 F F F 4 080-099 45 45 45 87 87 87 F F F 5 100-119 43 43 43 83 83 83 F F F 6 120-139 44 44 44 86 86 86 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 45 45 45 87 87 87 F F F 9 180-199 45 45 45 87 87 87 F F F 10 200-219 45 45 45 87 87 87 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 45 45 45 87 87 87 F F F 13 260-279 43 43 43 83 83 83 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 44 44 44 86 86 86 F F F 16 320-339 44 44 44 86 86 86 F F F 17 340-359 44 44 44 86 86 86 F F F 18 360-379 44 44 44 86 86 86 F F F 19 380-399 45 45 45 87 87 87 F F F 20 400-419 45 45 45 87 87 87 F F F 21 420-439 42 42 42 81 81 81 F F F 22 440-459 45 45 45 87 87 87 F F F 23 460-479 44 44 44 86 86 86 F F F 24 480-499 44 44 44 86 86 86 F F F 25 500-519 40 40 40 79 79 79 F F F 26 520-539 40 40 40 77 77 77 F F F 27 540-559 45 45 45 87 87 87 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 41 41 41 80 80 80 F F F Max gen: 45, Max flush: 87 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.134257909Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.178935012Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.179656023Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.179689362Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.179707372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.179722451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} {"msg":"Crucible b33ce313-ffb3-4cfe-a80f-3bd6799e3bc9 has session id: a6d25b4e-903d-43f6-b89f-0661c15400d6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.180413093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Upstairs opts: Upstairs UUID: b33ce313-ffb3-4cfe-a80f-3bd6799e3bc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.180444222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Crucible stats registered with UUID: b33ce313-ffb3-4cfe-a80f-3bd6799e3bc9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.180459012Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:53.180472122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.180976218Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.180999128Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} {"msg":"b33ce313-ffb3-4cfe-a80f-3bd6799e3bc9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181077135Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181093985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"0","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181116154Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"0","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181129624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"1","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181153104Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"1","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181178163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"2","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181205222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"2","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181234651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181262041Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181292Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35251)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181322959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51879)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181350628Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49254)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.181553483Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.182169947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"0","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.182202366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"2","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.18279676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"0","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.182957576Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"1","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.182985825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"2","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.183345476Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.183499872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.18357406Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"1","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[0]R flush_numbers[0..12]: [51, 84, 86, 86, 87, 83, 86, 75, 87, 87, 87, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184079916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[0]R generation[0..12]: [26, 44, 44, 44, 45, 43, 44, 38, 45, 45, 45, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184111675Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184127425Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[1]R flush_numbers[0..12]: [51, 84, 86, 86, 87, 83, 86, 75, 87, 87, 87, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184141415Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[1]R generation[0..12]: [26, 44, 44, 44, 45, 43, 44, 38, 45, 45, 45, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184154894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184168724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[2]R flush_numbers[0..12]: [51, 84, 86, 86, 87, 83, 86, 75, 87, 87, 87, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184182444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[2]R generation[0..12]: [26, 44, 44, 44, 45, 43, 44, 38, 45, 45, 45, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184195923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184209663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184222273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184235162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184247772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184260392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"":"downstairs","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184273271Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184285021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"0","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.18431046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"1","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.18432332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"client":"2","":"downstairs","session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.18433629Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184348489Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"b33ce313-ffb3-4cfe-a80f-3bd6799e3bc9 is now active with session: a6d25b4e-903d-43f6-b89f-0661c15400d6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184361039Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184372659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376,"session_id":"a6d25b4e-903d-43f6-b89f-0661c15400d6"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184384608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} {"msg":"Activated sub_volume b33ce313-ffb3-4cfe-a80f-3bd6799e3bc9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:00:53.184395528Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 245 len 40960 data: 4 4 7 7 6 7 5 5 3 3 03/30 Read block 108 len 28672 04/30 Read block 340 len 16384 05/30 Read block 163 len 8192 06/30 Write block 64 len 20480 data: 5 5 5 4 4 07/30 Write block 305 len 40960 data: 4 5 5 5 7 6 6 6 6 7 08/30 Read block 533 len 20480 09/30 Write block 407 len 20480 data: 4 4 4 5 5 10/30 Write block 534 len 40960 data: 6 6 6 6 6 7 5 4 6 9 11/30 Read block 99 len 4096 12/30 Flush 13/30 Write block 208 len 32768 data: 3 4 5 5 6 4 4 4 14/30 Write block 508 len 28672 data: 6 6 5 6 5 5 6 15/30 Read block 322 len 36864 16/30 Read block 539 len 32768 17/30 Read block 522 len 12288 18/30 Write block 428 len 12288 data: 5 4 5 19/30 Read block 591 len 8192 20/30 Write block 487 len 16384 data: 4 4 3 3 21/30 Write block 336 len 28672 data: 7 7 9 7 7 6 6 22/30 Write block 333 len 4096 data: 7 23/30 Write block 52 len 24576 data: 6 7 6 4 4 4 24/30 Write block 53 len 12288 data: 8 7 5 25/30 Write block 18 len 20480 data: 5 5 4 5 5 26/30 Read block 486 len 32768 27/30 Read block 49 len 16384 28/30 Write block 105 len 4096 data: 6 29/30 Write block 460 len 20480 data: 3 3 3 3 2 30/30 Read block 234 len 40960 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 16 18 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:00:53.204112939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 51 51 51 T T T 1 020-039 44 44 44 84 84 84 T T T 2 040-059 44 44 44 86 86 86 T T T 3 060-079 46 46 46 89 89 89 F F F 4 080-099 45 45 45 87 87 87 F F F 5 100-119 43 43 43 83 83 83 T T T 6 120-139 44 44 44 86 86 86 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 45 45 45 87 87 87 F F F 9 180-199 45 45 45 87 87 87 F F F 10 200-219 45 45 45 87 87 87 T T T 11 220-239 27 27 27 56 56 56 F F F 12 240-259 46 46 46 89 89 89 F F F 13 260-279 43 43 43 83 83 83 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 46 46 46 89 89 89 F F F 16 320-339 44 44 44 86 86 86 T T T 17 340-359 44 44 44 86 86 86 T T T 18 360-379 44 44 44 86 86 86 F F F 19 380-399 45 45 45 87 87 87 F F F 20 400-419 46 46 46 89 89 89 F F F 21 420-439 42 42 42 81 81 81 T T T 22 440-459 45 45 45 87 87 87 F F F 23 460-479 44 44 44 86 86 86 T T T 24 480-499 44 44 44 86 86 86 T T T 25 500-519 40 40 40 79 79 79 T T T 26 520-539 46 46 46 89 89 89 F F F 27 540-559 46 46 46 89 89 89 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 41 41 41 80 80 80 F F F Max gen: 46, Max flush: 89 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 47 47 47 90 90 90 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 46 46 46 89 89 89 F F F 4 080-099 45 45 45 87 87 87 F F F 5 100-119 47 47 47 90 90 90 F F F 6 120-139 44 44 44 86 86 86 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 45 45 45 87 87 87 F F F 9 180-199 45 45 45 87 87 87 F F F 10 200-219 47 47 47 90 90 90 F F F 11 220-239 27 27 27 56 56 56 F F F 12 240-259 46 46 46 89 89 89 F F F 13 260-279 43 43 43 83 83 83 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 46 46 46 89 89 89 F F F 16 320-339 47 47 47 90 90 90 F F F 17 340-359 47 47 47 90 90 90 F F F 18 360-379 44 44 44 86 86 86 F F F 19 380-399 45 45 45 87 87 87 F F F 20 400-419 46 46 46 89 89 89 F F F 21 420-439 47 47 47 90 90 90 F F F 22 440-459 45 45 45 87 87 87 F F F 23 460-479 47 47 47 90 90 90 F F F 24 480-499 47 47 47 90 90 90 F F F 25 500-519 47 47 47 90 90 90 F F F 26 520-539 46 46 46 89 89 89 F F F 27 540-559 46 46 46 89 89 89 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 41 41 41 80 80 80 F F F Max gen: 47, Max flush: 90 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.470306711Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.513923742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.514676342Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.514714291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.51472848Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.51474311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} {"msg":"Crucible 6294264c-e184-44e9-a848-fda2d441b2d6 has session id: 21317e85-0e49-4843-a225-ecde55068c85","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.515272676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Upstairs opts: Upstairs UUID: 6294264c-e184-44e9-a848-fda2d441b2d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.515318155Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Crucible stats registered with UUID: 6294264c-e184-44e9-a848-fda2d441b2d6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.515342864Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:07.515364714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.51587681Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.51590467Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} {"msg":"6294264c-e184-44e9-a848-fda2d441b2d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.515950428Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.515973388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"0","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.515996567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"0","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516010607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516025086Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516038306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"2","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516052146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"2","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516065235Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516083645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516109554Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"ds_connection connected from Ok(127.0.0.1:60156)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516151903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"ds_connection connected from Ok(127.0.0.1:63187)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516179492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:07.516230261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:07.51626473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:07.516300199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516446025Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516933062Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"2","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.516963922Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"0","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.517614064Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"0","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.517748771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"2","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.518134151Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:07.518285987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.517984138Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"ds_connection connected from Ok(127.0.0.1:54422)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.518063326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.519180806Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.519840989Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[0]R flush_numbers[0..12]: [90, 90, 90, 89, 87, 90, 86, 75, 87, 87, 90, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520509791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[0]R generation[0..12]: [47, 47, 47, 46, 45, 47, 44, 38, 45, 45, 47, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520538571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.52055956Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[1]R flush_numbers[0..12]: [90, 90, 90, 89, 87, 90, 86, 75, 87, 87, 90, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.52057455Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[1]R generation[0..12]: [47, 47, 47, 46, 45, 47, 44, 38, 45, 45, 47, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520588519Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520602399Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[2]R flush_numbers[0..12]: [90, 90, 90, 89, 87, 90, 86, 75, 87, 87, 90, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520615818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[2]R generation[0..12]: [47, 47, 47, 46, 45, 47, 44, 38, 45, 45, 47, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520629058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520642618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520655848Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520669157Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520681637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520694167Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"":"downstairs","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520707186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520718996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"0","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520732315Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"1","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520745205Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"client":"2","":"downstairs","session_id":"21317e85-0e49-4843-a225-ecde55068c85"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520764294Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520776604Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"6294264c-e184-44e9-a848-fda2d441b2d6 is now active with session: 21317e85-0e49-4843-a225-ecde55068c85","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520789124Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520800754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384,"session_id":"21317e85-0e49-4843-a225-ecde55068c85"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520812443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} {"msg":"Activated sub_volume 6294264c-e184-44e9-a848-fda2d441b2d6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:17.520823483Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 36864 data: 8 8 9 8 7 8 6 7 4 02/30 Write block 206 len 28672 data: 4 4 4 5 6 6 7 03/30 Write block 394 len 40960 data: 7 6 5 5 8 6 7 7 5 5 04/30 Read block 207 len 12288 05/30 Read block 503 len 24576 06/30 Flush 07/30 Write block 276 len 16384 data: 3 3 3 3 08/30 Read block 376 len 12288 09/30 Write block 87 len 32768 data: 6 7 9 7 7 6 8 7 10/30 Write block 519 len 24576 data: 3 2 2 2 2 3 11/30 Read block 138 len 12288 12/30 Write block 357 len 28672 data: 2 3 3 5 5 7 6 13/30 Read block 431 len 36864 14/30 Read block 251 len 4096 15/30 Flush 16/30 Write block 370 len 16384 data: 4 3 4 4 17/30 Write block 145 len 32768 data: 4 3 3 3 3 3 4 4 18/30 Write block 536 len 16384 data: 7 7 7 8 19/30 Read block 123 len 4096 20/30 Flush 21/30 Flush 22/30 Write block 28 len 24576 data: 10 10 9 7 7 6 23/30 Write block 191 len 8192 data: 6 6 24/30 Write block 153 len 24576 data: 3 3 5 5 5 4 25/30 Read block 460 len 4096 26/30 Flush 27/30 Write block 587 len 4096 data: 6 28/30 Write block 224 len 24576 data: 5 4 6 7 6 5 29/30 Read block 65 len 24576 30/30 Write block 395 len 12288 data: 7 6 6 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 1 5 Done 3 1 3 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:17.541218096Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 48 48 48 95 95 95 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 46 46 46 89 89 89 F F F 4 080-099 48 48 48 92 92 92 F F F 5 100-119 48 48 48 91 91 91 F F F 6 120-139 44 44 44 86 86 86 F F F 7 140-159 48 48 48 95 95 95 F F F 8 160-179 45 45 45 87 87 87 F F F 9 180-199 48 48 48 95 95 95 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 27 27 27 56 56 56 T T T 12 240-259 46 46 46 89 89 89 F F F 13 260-279 48 48 48 92 92 92 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 46 46 46 89 89 89 F F F 16 320-339 47 47 47 90 90 90 F F F 17 340-359 48 48 48 92 92 92 F F F 18 360-379 48 48 48 93 93 93 F F F 19 380-399 48 48 48 91 91 91 T T T 20 400-419 48 48 48 91 91 91 F F F 21 420-439 47 47 47 90 90 90 F F F 22 440-459 45 45 45 87 87 87 F F F 23 460-479 47 47 47 90 90 90 F F F 24 480-499 47 47 47 90 90 90 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 48 48 48 93 93 93 F F F 27 540-559 46 46 46 89 89 89 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 41 41 41 80 80 80 T T T Max gen: 48, Max flush: 95 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 48 48 48 95 95 95 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 46 46 46 89 89 89 F F F 4 080-099 48 48 48 92 92 92 F F F 5 100-119 48 48 48 91 91 91 F F F 6 120-139 44 44 44 86 86 86 F F F 7 140-159 48 48 48 95 95 95 F F F 8 160-179 45 45 45 87 87 87 F F F 9 180-199 48 48 48 95 95 95 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 49 49 49 96 96 96 F F F 12 240-259 46 46 46 89 89 89 F F F 13 260-279 48 48 48 92 92 92 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 46 46 46 89 89 89 F F F 16 320-339 47 47 47 90 90 90 F F F 17 340-359 48 48 48 92 92 92 F F F 18 360-379 48 48 48 93 93 93 F F F 19 380-399 49 49 49 96 96 96 F F F 20 400-419 48 48 48 91 91 91 F F F 21 420-439 47 47 47 90 90 90 F F F 22 440-459 45 45 45 87 87 87 F F F 23 460-479 47 47 47 90 90 90 F F F 24 480-499 47 47 47 90 90 90 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 48 48 48 93 93 93 F F F 27 540-559 46 46 46 89 89 89 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 49 49 49 96 96 96 F F F Max gen: 49, Max flush: 96 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.775771988Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.819384288Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.820289934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.820323823Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.820337413Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.820351503Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} {"msg":"Crucible cf55dee5-a8b4-4666-b100-4ee21d6d7b43 has session id: bbdf5dd7-ad47-4ce3-a872-0e6601494084","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821066704Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Upstairs opts: Upstairs UUID: cf55dee5-a8b4-4666-b100-4ee21d6d7b43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821113933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Crucible stats registered with UUID: cf55dee5-a8b4-4666-b100-4ee21d6d7b43","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821138102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:21.821158411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82158443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821619739Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} {"msg":"cf55dee5-a8b4-4666-b100-4ee21d6d7b43 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821651728Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821667468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"0","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821698897Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"0","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821726536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"1","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821760365Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"1","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821790085Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"2","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821818124Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"2","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821843473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821873302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821900602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"ds_connection connected from Ok(127.0.0.1:34152)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.821925141Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"ds_connection connected from Ok(127.0.0.1:42940)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82194968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"ds_connection connected from Ok(127.0.0.1:55257)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82197569Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82270748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"2","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.822747739Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"0","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82348015Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"0","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.823538089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"2","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.824118663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"1","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.824206901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82423642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.824717938Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"1","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[0]R flush_numbers[0..12]: [90, 95, 90, 89, 92, 91, 86, 95, 87, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825319552Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[0]R generation[0..12]: [47, 48, 47, 46, 48, 48, 44, 48, 45, 48, 48, 49]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825339311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825353761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[1]R flush_numbers[0..12]: [90, 95, 90, 89, 92, 91, 86, 95, 87, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825367621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[1]R generation[0..12]: [47, 48, 47, 46, 48, 48, 44, 48, 45, 48, 48, 49]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82538111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.82539499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[2]R flush_numbers[0..12]: [90, 95, 90, 89, 92, 91, 86, 95, 87, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825408199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[2]R generation[0..12]: [47, 48, 47, 46, 48, 48, 44, 48, 45, 48, 48, 49]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825421769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825435679Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825448288Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825461068Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825473628Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825489487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"":"downstairs","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825502667Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-03-19T17:01:21.825521006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"0","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825534416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"1","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825547146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"client":"2","":"downstairs","session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825560095Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825572175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"cf55dee5-a8b4-4666-b100-4ee21d6d7b43 is now active with session: bbdf5dd7-ad47-4ce3-a872-0e6601494084","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825584815Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825596484Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392,"session_id":"bbdf5dd7-ad47-4ce3-a872-0e6601494084"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825608474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} {"msg":"Activated sub_volume cf55dee5-a8b4-4666-b100-4ee21d6d7b43","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:21.825619414Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 344 len 8192 data: 7 7 03/30 Write block 261 len 16384 data: 6 6 4 4 04/30 Write block 110 len 20480 data: 3 4 4 4 4 05/30 Read block 279 len 24576 06/30 Write block 538 len 8192 data: 8 9 07/30 Write block 482 len 32768 data: 4 4 4 4 5 5 5 4 08/30 Flush 09/30 Write block 436 len 32768 data: 4 4 3 3 3 4 5 4 10/30 Read block 349 len 20480 11/30 Read block 337 len 4096 12/30 Write block 259 len 16384 data: 5 5 7 7 13/30 Write block 22 len 40960 data: 6 5 6 7 7 7 11 11 10 8 14/30 Read block 583 len 32768 15/30 Read block 216 len 20480 16/30 Read block 422 len 8192 17/30 Read block 19 len 36864 18/30 Write block 101 len 28672 data: 9 8 9 7 8 5 4 19/30 Read block 179 len 20480 20/30 Read block 116 len 28672 21/30 Flush 22/30 Write block 224 len 32768 data: 6 5 7 8 7 6 5 5 23/30 Write block 590 len 20480 data: 2 2 2 2 2 24/30 Read block 376 len 8192 25/30 Read block 561 len 36864 26/30 Write block 391 len 20480 data: 6 6 6 8 8 27/30 Read block 494 len 32768 28/30 Write block 397 len 32768 data: 7 9 7 8 8 6 6 4 29/30 Read block 94 len 16384 30/30 Write block 438 len 12288 data: 4 4 4 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:21.846663049Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 50 50 50 99 99 99 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 46 46 46 89 89 89 F F F 4 080-099 48 48 48 92 92 92 F F F 5 100-119 50 50 50 99 99 99 F F F 6 120-139 44 44 44 86 86 86 F F F 7 140-159 48 48 48 95 95 95 F F F 8 160-179 45 45 45 87 87 87 F F F 9 180-199 48 48 48 95 95 95 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 49 49 49 96 96 96 T T T 12 240-259 50 50 50 99 99 99 F F F 13 260-279 50 50 50 99 99 99 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 46 46 46 89 89 89 F F F 16 320-339 47 47 47 90 90 90 F F F 17 340-359 50 50 50 98 98 98 F F F 18 360-379 48 48 48 93 93 93 F F F 19 380-399 49 49 49 96 96 96 T T T 20 400-419 48 48 48 91 91 91 T T T 21 420-439 50 50 50 99 99 99 T T T 22 440-459 50 50 50 99 99 99 T T T 23 460-479 47 47 47 90 90 90 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 50 50 50 98 98 98 F F F 27 540-559 46 46 46 89 89 89 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 49 49 49 96 96 96 T T T Max gen: 50, Max flush: 99 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 50 50 50 99 99 99 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 46 46 46 89 89 89 F F F 4 080-099 48 48 48 92 92 92 F F F 5 100-119 50 50 50 99 99 99 F F F 6 120-139 44 44 44 86 86 86 F F F 7 140-159 48 48 48 95 95 95 F F F 8 160-179 45 45 45 87 87 87 F F F 9 180-199 48 48 48 95 95 95 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 51 51 51 100 100 100 F F F 12 240-259 50 50 50 99 99 99 F F F 13 260-279 50 50 50 99 99 99 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 46 46 46 89 89 89 F F F 16 320-339 47 47 47 90 90 90 F F F 17 340-359 50 50 50 98 98 98 F F F 18 360-379 48 48 48 93 93 93 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 51 51 51 100 100 100 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 51 51 51 100 100 100 F F F 23 460-479 47 47 47 90 90 90 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 48 48 48 92 92 92 F F F 26 520-539 50 50 50 98 98 98 F F F 27 540-559 46 46 46 89 89 89 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 51 51 51 100 100 100 F F F Max gen: 51, Max flush: 100 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.09264743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:26.13742159Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.13783595Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.170008842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.170671774Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.170706433Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.170720343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.170735063Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} {"msg":"Crucible 4e8957f8-d702-4fd5-b3b8-995a319e7c50 has session id: 96e1bc9f-d4b2-4743-8538-8d1365f6cc08","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.171323987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Upstairs opts: Upstairs UUID: 4e8957f8-d702-4fd5-b3b8-995a319e7c50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.171364306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Crucible stats registered with UUID: 4e8957f8-d702-4fd5-b3b8-995a319e7c50","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.171384876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:26.171398225Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.171842344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.171865863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} {"msg":"4e8957f8-d702-4fd5-b3b8-995a319e7c50 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.171935511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.171954581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"0","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.17197757Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"0","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.17199171Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"1","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172006199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"1","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172026499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"2","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172052198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"2","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172077617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172098897Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172113207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"ds_connection connected from Ok(127.0.0.1:51973)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172235033Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"ds_connection connected from Ok(127.0.0.1:48307)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172338261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"ds_connection connected from Ok(127.0.0.1:49370)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172379909Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.172965394Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"2","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.17312527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"1","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.173332164Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"0","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.173677695Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"1","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.173765173Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"2","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174037926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"0","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174359017Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174381457Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[0]R flush_numbers[0..12]: [90, 99, 90, 89, 92, 99, 86, 95, 87, 95, 91, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174690339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[0]R generation[0..12]: [47, 50, 47, 46, 48, 50, 44, 48, 45, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174713328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174727678Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[1]R flush_numbers[0..12]: [90, 99, 90, 89, 92, 99, 86, 95, 87, 95, 91, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174740987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[1]R generation[0..12]: [47, 50, 47, 46, 48, 50, 44, 48, 45, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174754187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174770697Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[2]R flush_numbers[0..12]: [90, 99, 90, 89, 92, 99, 86, 95, 87, 95, 91, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174784196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[2]R generation[0..12]: [47, 50, 47, 46, 48, 50, 44, 48, 45, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174797636Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174811275Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174823715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174836265Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174848644Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174861074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"":"downstairs","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174873674Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174885243Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"0","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174897903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"1","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174910553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"client":"2","":"downstairs","session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:01:26.174923113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174941182Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"4e8957f8-d702-4fd5-b3b8-995a319e7c50 is now active with session: 96e1bc9f-d4b2-4743-8538-8d1365f6cc08","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174953612Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174967261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400,"session_id":"96e1bc9f-d4b2-4743-8538-8d1365f6cc08"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174979371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} {"msg":"Activated sub_volume 4e8957f8-d702-4fd5-b3b8-995a319e7c50","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:26.174990241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 504 len 32768 data: 3 3 3 4 7 7 6 7 03/30 Write block 132 len 40960 data: 4 4 4 4 4 6 6 5 4 4 04/30 Write block 350 len 40960 data: 5 6 4 3 3 3 3 3 4 4 05/30 Read block 163 len 4096 06/30 Read block 309 len 16384 07/30 Read block 587 len 40960 08/30 Write block 314 len 16384 data: 8 6 6 5 09/30 Read block 380 len 32768 10/30 Read block 124 len 32768 11/30 Write block 553 len 12288 data: 10 8 8 12/30 Read block 307 len 32768 13/30 Write block 583 len 12288 data: 4 3 3 14/30 Read block 62 len 28672 15/30 Write block 179 len 20480 data: 5 4 3 2 4 16/30 Read block 550 len 28672 17/30 Read block 75 len 4096 18/30 Write block 102 len 36864 data: 9 10 8 9 6 5 4 4 4 19/30 Read block 216 len 40960 20/30 Read block 205 len 20480 21/30 Read block 488 len 12288 22/30 Write block 367 len 32768 data: 4 4 3 5 4 5 5 4 23/30 Read block 373 len 4096 24/30 Read block 40 len 32768 25/30 Read block 546 len 16384 26/30 Flush 27/30 Write block 472 len 24576 data: 3 5 5 5 5 5 28/30 Read block 117 len 20480 29/30 Write block 175 len 4096 data: 5 30/30 Read block 549 len 8192 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:26.19703592Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 50 50 50 99 99 99 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 46 46 46 89 89 89 F F F 4 080-099 48 48 48 92 92 92 F F F 5 100-119 52 52 52 102 102 102 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 52 52 52 102 102 102 T T T 9 180-199 52 52 52 102 102 102 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 51 51 51 100 100 100 F F F 12 240-259 50 50 50 99 99 99 F F F 13 260-279 50 50 50 99 99 99 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 52 52 52 102 102 102 F F F 16 320-339 47 47 47 90 90 90 F F F 17 340-359 52 52 52 102 102 102 F F F 18 360-379 52 52 52 102 102 102 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 51 51 51 100 100 100 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 51 51 51 100 100 100 F F F 23 460-479 47 47 47 90 90 90 T T T 24 480-499 50 50 50 98 98 98 F F F 25 500-519 52 52 52 102 102 102 F F F 26 520-539 50 50 50 98 98 98 F F F 27 540-559 52 52 52 102 102 102 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 52, Max flush: 102 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 50 50 50 99 99 99 F F F 2 040-059 47 47 47 90 90 90 F F F 3 060-079 46 46 46 89 89 89 F F F 4 080-099 48 48 48 92 92 92 F F F 5 100-119 52 52 52 102 102 102 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 53 53 53 103 103 103 F F F 9 180-199 52 52 52 102 102 102 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 51 51 51 100 100 100 F F F 12 240-259 50 50 50 99 99 99 F F F 13 260-279 50 50 50 99 99 99 F F F 14 280-299 40 40 40 77 77 77 F F F 15 300-319 52 52 52 102 102 102 F F F 16 320-339 47 47 47 90 90 90 F F F 17 340-359 52 52 52 102 102 102 F F F 18 360-379 52 52 52 102 102 102 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 51 51 51 100 100 100 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 51 51 51 100 100 100 F F F 23 460-479 53 53 53 103 103 103 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 52 52 52 102 102 102 F F F 26 520-539 50 50 50 98 98 98 F F F 27 540-559 52 52 52 102 102 102 F F F 28 560-579 45 45 45 87 87 87 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 53, Max flush: 103 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.450649469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.494136013Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.494869454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.494905363Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.494919323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.494938002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} {"msg":"Crucible b6282b62-c38e-4365-857d-9f4625c38bd0 has session id: f30fc34b-705b-49a3-9e20-00d98344bbb9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.49541108Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Upstairs opts: Upstairs UUID: b6282b62-c38e-4365-857d-9f4625c38bd0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.495579575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Crucible stats registered with UUID: b6282b62-c38e-4365-857d-9f4625c38bd0","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.495600205Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:30.495613574Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496048583Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496081392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} {"msg":"b6282b62-c38e-4365-857d-9f4625c38bd0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.4961375Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496180359Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"0","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496202689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"0","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496216969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"1","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496234238Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"1","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496257497Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496284097Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496351525Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496382754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496408913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:30.496437023Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47077)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496464162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:30.496492291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:30.49652294Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.49654779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53938)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.496699886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.497122025Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"0","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.49728461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"1","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.49766712Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"0","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.497809307Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"1","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.498246125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:30.498354472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.498120082Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40343)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.4982097Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.49935178Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.499997783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[0]R flush_numbers[0..12]: [90, 99, 90, 89, 92, 102, 102, 102, 103, 102, 91, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500615896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[0]R generation[0..12]: [47, 50, 47, 46, 48, 52, 52, 52, 53, 52, 48, 51]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500643896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500659955Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[1]R flush_numbers[0..12]: [90, 99, 90, 89, 92, 102, 102, 102, 103, 102, 91, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500673895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[1]R generation[0..12]: [47, 50, 47, 46, 48, 52, 52, 52, 53, 52, 48, 51]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500688115Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500702324Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[2]R flush_numbers[0..12]: [90, 99, 90, 89, 92, 102, 102, 102, 103, 102, 91, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500715634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[2]R generation[0..12]: [47, 50, 47, 46, 48, 52, 52, 52, 53, 52, 48, 51]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500728713Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500742473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500759563Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500772852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500785252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500797912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"":"downstairs","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500810691Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500822351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"0","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500835381Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"1","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.50085542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"client":"2","":"downstairs","session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.50087723Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500889689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"b6282b62-c38e-4365-857d-9f4625c38bd0 is now active with session: f30fc34b-705b-49a3-9e20-00d98344bbb9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500902309Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500914199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408,"session_id":"f30fc34b-705b-49a3-9e20-00d98344bbb9"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500926178Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} {"msg":"Activated sub_volume b6282b62-c38e-4365-857d-9f4625c38bd0","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:40.500937018Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 39 len 32768 data: 5 6 6 6 6 6 6 3 02/30 Flush 03/30 Write block 273 len 8192 data: 5 5 04/30 Read block 345 len 36864 05/30 Write block 57 len 4096 data: 5 06/30 Write block 281 len 16384 data: 4 4 4 4 07/30 Write block 109 len 8192 data: 5 5 08/30 Write block 562 len 32768 data: 4 5 7 7 8 7 8 6 09/30 Write block 319 len 8192 data: 5 5 10/30 Write block 66 len 8192 data: 6 5 11/30 Read block 80 len 12288 12/30 Write block 508 len 8192 data: 8 8 13/30 Read block 587 len 4096 14/30 Flush 15/30 Write block 359 len 20480 data: 5 6 6 8 7 16/30 Read block 259 len 20480 17/30 Write block 262 len 24576 data: 8 5 5 5 5 5 18/30 Read block 257 len 28672 19/30 Read block 248 len 36864 20/30 Read block 457 len 8192 21/30 Flush 22/30 Write block 93 len 16384 data: 9 8 7 9 23/30 Read block 436 len 28672 24/30 Read block 382 len 16384 25/30 Write block 450 len 8192 data: 3 5 26/30 Flush 27/30 Write block 274 len 28672 data: 6 4 4 4 4 4 3 28/30 Read block 329 len 12288 29/30 Write block 511 len 16384 data: 8 6 6 7 30/30 Write block 234 len 40960 data: 3 5 5 5 4 4 4 5 6 7 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:40.520156362Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 54 54 54 104 104 104 F F F 2 040-059 54 54 54 105 105 105 F F F 3 060-079 54 54 54 105 105 105 F F F 4 080-099 54 54 54 107 107 107 F F F 5 100-119 54 54 54 105 105 105 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 53 53 53 103 103 103 F F F 9 180-199 52 52 52 102 102 102 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 51 51 51 100 100 100 T T T 12 240-259 50 50 50 99 99 99 T T T 13 260-279 54 54 54 106 106 106 T T T 14 280-299 54 54 54 105 105 105 T T T 15 300-319 54 54 54 105 105 105 F F F 16 320-339 54 54 54 105 105 105 F F F 17 340-359 54 54 54 106 106 106 F F F 18 360-379 54 54 54 106 106 106 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 51 51 51 100 100 100 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 54 54 54 107 107 107 F F F 23 460-479 53 53 53 103 103 103 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 54 54 54 105 105 105 T T T 26 520-539 50 50 50 98 98 98 F F F 27 540-559 52 52 52 102 102 102 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 54, Max flush: 107 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 54 54 54 104 104 104 F F F 2 040-059 54 54 54 105 105 105 F F F 3 060-079 54 54 54 105 105 105 F F F 4 080-099 54 54 54 107 107 107 F F F 5 100-119 54 54 54 105 105 105 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 53 53 53 103 103 103 F F F 9 180-199 52 52 52 102 102 102 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 55 55 55 108 108 108 F F F 12 240-259 55 55 55 108 108 108 F F F 13 260-279 55 55 55 108 108 108 F F F 14 280-299 55 55 55 108 108 108 F F F 15 300-319 54 54 54 105 105 105 F F F 16 320-339 54 54 54 105 105 105 F F F 17 340-359 54 54 54 106 106 106 F F F 18 360-379 54 54 54 106 106 106 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 51 51 51 100 100 100 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 54 54 54 107 107 107 F F F 23 460-479 53 53 53 103 103 103 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 55 55 55 108 108 108 F F F 26 520-539 50 50 50 98 98 98 F F F 27 540-559 52 52 52 102 102 102 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 55, Max flush: 108 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.760412933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.804445343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.805142985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.805178204Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.805192134Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.805206123Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} {"msg":"Crucible a3fa0264-3c2f-47a0-986e-9f125ec4dd24 has session id: c2226d9a-39e4-486d-944c-f4b8f133ca77","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.805848566Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Upstairs opts: Upstairs UUID: a3fa0264-3c2f-47a0-986e-9f125ec4dd24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.805882146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Crucible stats registered with UUID: a3fa0264-3c2f-47a0-986e-9f125ec4dd24","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.805897295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:44.805914115Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806345433Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806368793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} {"msg":"a3fa0264-3c2f-47a0-986e-9f125ec4dd24 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806454311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.80647216Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"0","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806494639Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"0","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806508089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"1","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806522029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"1","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806534888Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"2","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806548428Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"2","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806561068Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806618666Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806642666Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"ds_connection connected from Ok(127.0.0.1:38250)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806663755Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"ds_connection connected from Ok(127.0.0.1:41440)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806687054Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"ds_connection connected from Ok(127.0.0.1:63512)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.806716953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.807456024Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"1","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.807491853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"0","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.808265143Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"0","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.808320131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"1","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.808859207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.808992613Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.809095701Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"2","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.809622487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"2","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[0]R flush_numbers[0..12]: [90, 104, 105, 105, 107, 105, 102, 102, 103, 102, 91, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810144343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[0]R generation[0..12]: [47, 54, 54, 54, 54, 54, 52, 52, 53, 52, 48, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810163033Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810177322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[1]R flush_numbers[0..12]: [90, 104, 105, 105, 107, 105, 102, 102, 103, 102, 91, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810190812Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[1]R generation[0..12]: [47, 54, 54, 54, 54, 54, 52, 52, 53, 52, 48, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810204172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810222081Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[2]R flush_numbers[0..12]: [90, 104, 105, 105, 107, 105, 102, 102, 103, 102, 91, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810236101Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[2]R generation[0..12]: [47, 54, 54, 54, 54, 54, 52, 52, 53, 52, 48, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810249911Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.81026366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.81027646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810289339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810302159Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810315019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"":"downstairs","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810327828Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810346158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"0","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810358957Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"1","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810371617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"client":"2","":"downstairs","session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810400527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810414816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"a3fa0264-3c2f-47a0-986e-9f125ec4dd24 is now active with session: c2226d9a-39e4-486d-944c-f4b8f133ca77","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810427546Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810439235Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416,"session_id":"c2226d9a-39e4-486d-944c-f4b8f133ca77"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810452975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} {"msg":"Activated sub_volume a3fa0264-3c2f-47a0-986e-9f125ec4dd24","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:44.810464305Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 413 len 20480 data: 6 4 5 6 6 02/30 Read block 408 len 28672 03/30 Read block 372 len 28672 04/30 Read block 356 len 4096 05/30 Read block 228 len 28672 06/30 Write block 192 len 4096 data: 7 07/30 Write block 49 len 4096 data: 4 08/30 Write block 114 len 8192 data: 5 3 09/30 Read block 143 len 8192 10/30 Write block 523 len 4096 data: 3 11/30 Read block 580 len 24576 12/30 Read block 515 len 4096 13/30 Read block 59 len 28672 14/30 Write block 349 len 4096 data: 5 15/30 Read block 48 len 20480 16/30 Write block 149 len 12288 data: 4 4 5 17/30 Read block 134 len 8192 18/30 Flush 19/30 Write block 157 len 12288 data: 6 5 4 20/30 Write block 310 len 36864 data: 7 7 7 7 9 7 7 6 4 21/30 Read block 545 len 24576 22/30 Read block 595 len 12288 23/30 Write block 452 len 16384 data: 5 5 6 6 24/30 Read block 579 len 20480 25/30 Read block 85 len 20480 26/30 Write block 461 len 28672 data: 4 4 4 3 2 3 2 27/30 Read block 120 len 20480 28/30 Write block 526 len 32768 data: 3 4 4 4 4 5 6 5 29/30 Write block 169 len 32768 data: 5 6 6 6 6 5 6 5 30/30 Write block 349 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:44.828991717Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 54 54 54 104 104 104 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 54 54 54 105 105 105 F F F 4 080-099 54 54 54 107 107 107 F F F 5 100-119 56 56 56 109 109 109 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 56 56 56 109 109 109 T T T 8 160-179 53 53 53 103 103 103 T T T 9 180-199 56 56 56 109 109 109 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 55 55 55 108 108 108 F F F 12 240-259 55 55 55 108 108 108 F F F 13 260-279 55 55 55 108 108 108 F F F 14 280-299 55 55 55 108 108 108 F F F 15 300-319 54 54 54 105 105 105 T T T 16 320-339 54 54 54 105 105 105 F F F 17 340-359 56 56 56 109 109 109 T T T 18 360-379 54 54 54 106 106 106 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 56 56 56 109 109 109 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 54 54 54 107 107 107 T T T 23 460-479 53 53 53 103 103 103 T T T 24 480-499 50 50 50 98 98 98 F F F 25 500-519 55 55 55 108 108 108 F F F 26 520-539 56 56 56 109 109 109 T T T 27 540-559 52 52 52 102 102 102 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 56, Max flush: 109 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 54 54 54 104 104 104 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 54 54 54 105 105 105 F F F 4 080-099 54 54 54 107 107 107 F F F 5 100-119 56 56 56 109 109 109 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 57 57 57 110 110 110 F F F 8 160-179 57 57 57 110 110 110 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 48 48 48 91 91 91 F F F 11 220-239 55 55 55 108 108 108 F F F 12 240-259 55 55 55 108 108 108 F F F 13 260-279 55 55 55 108 108 108 F F F 14 280-299 55 55 55 108 108 108 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 54 54 54 105 105 105 F F F 17 340-359 57 57 57 110 110 110 F F F 18 360-379 54 54 54 106 106 106 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 56 56 56 109 109 109 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 57 57 57 110 110 110 F F F 23 460-479 57 57 57 110 110 110 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 55 55 55 108 108 108 F F F 26 520-539 57 57 57 110 110 110 F F F 27 540-559 52 52 52 102 102 102 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 57, Max flush: 110 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.076664273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:49.122502416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.122877026Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.154288118Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.154924001Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.1549635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15498328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15499809Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} {"msg":"Crucible c3bd6245-a88c-42e4-b8c6-d37fea095852 has session id: d2b4d1ff-3f48-4939-833c-a50cd19aa4c4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.155593584Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Upstairs opts: Upstairs UUID: c3bd6245-a88c-42e4-b8c6-d37fea095852, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.155639403Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Crucible stats registered with UUID: c3bd6245-a88c-42e4-b8c6-d37fea095852","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.155662582Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:49.155681582Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15610399Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15612376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} {"msg":"c3bd6245-a88c-42e4-b8c6-d37fea095852 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156198578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156215747Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"0","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156238367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"0","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156251897Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"1","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156265876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"1","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156282596Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"2","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156307595Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"2","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156330955Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156353994Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156377743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42294)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156402473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:39405)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156445821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44225)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.156473251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.157203801Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"2","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15723978Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"1","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15726741Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"0","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.157869344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"2","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15801845Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"1","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158047749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"0","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158431249Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158577985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[0]R flush_numbers[0..12]: [90, 104, 109, 105, 107, 109, 102, 110, 110, 109, 91, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158689642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[0]R generation[0..12]: [47, 54, 56, 54, 54, 56, 52, 57, 57, 56, 48, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158708952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158723232Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[1]R flush_numbers[0..12]: [90, 104, 109, 105, 107, 109, 102, 110, 110, 109, 91, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158737081Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[1]R generation[0..12]: [47, 54, 56, 54, 54, 56, 52, 57, 57, 56, 48, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158750541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15876412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[2]R flush_numbers[0..12]: [90, 104, 109, 105, 107, 109, 102, 110, 110, 109, 91, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15877769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[2]R generation[0..12]: [47, 54, 56, 54, 54, 56, 52, 57, 57, 56, 48, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.15879093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158804319Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158817329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158830139Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158842928Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158858948Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"":"downstairs","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158878947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158890737Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"0","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158903607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"1","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158916306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"client":"2","":"downstairs","session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158929036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158941046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"c3bd6245-a88c-42e4-b8c6-d37fea095852 is now active with session: d2b4d1ff-3f48-4939-833c-a50cd19aa4c4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158953655Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158965425Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424,"session_id":"d2b4d1ff-3f48-4939-833c-a50cd19aa4c4"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158977215Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} {"msg":"Activated sub_volume c3bd6245-a88c-42e4-b8c6-d37fea095852","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:49.158988494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 275 len 24576 data: 5 5 5 5 5 4 02/30 Write block 449 len 20480 data: 2 4 6 6 6 03/30 Read block 291 len 32768 04/30 Write block 514 len 40960 data: 8 4 4 4 3 4 3 3 3 4 05/30 Write block 33 len 16384 data: 7 7 7 7 06/30 Read block 345 len 24576 07/30 Write block 366 len 36864 data: 4 5 5 4 6 5 6 6 5 08/30 Read block 162 len 8192 09/30 Write block 410 len 28672 data: 6 6 6 7 5 6 7 10/30 Write block 467 len 36864 data: 3 2 3 4 3 4 6 6 6 11/30 Write block 30 len 4096 data: 11 12/30 Write block 342 len 32768 data: 7 7 8 8 6 5 6 7 13/30 Read block 248 len 8192 14/30 Flush 15/30 Flush 16/30 Flush 17/30 Write block 97 len 4096 data: 9 18/30 Write block 247 len 36864 data: 8 8 7 8 6 6 4 4 3 19/30 Write block 135 len 28672 data: 5 5 7 7 6 5 5 20/30 Read block 198 len 16384 21/30 Write block 359 len 12288 data: 6 7 7 22/30 Write block 267 len 36864 data: 6 6 5 5 4 4 6 7 6 23/30 Write block 74 len 40960 data: 4 6 6 6 5 5 5 5 4 7 24/30 Write block 27 len 4096 data: 8 25/30 Read block 197 len 24576 26/30 Flush 27/30 Write block 214 len 16384 data: 5 5 4 4 28/30 Write block 362 len 36864 data: 9 8 5 4 5 6 6 5 7 29/30 Read block 319 len 24576 30/30 Read block 425 len 40960 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:49.179483685Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 58 58 58 114 114 114 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 58 58 58 114 114 114 F F F 4 080-099 58 58 58 114 114 114 F F F 5 100-119 56 56 56 109 109 109 F F F 6 120-139 58 58 58 114 114 114 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 57 57 57 110 110 110 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 48 48 48 91 91 91 T T T 11 220-239 55 55 55 108 108 108 F F F 12 240-259 58 58 58 114 114 114 F F F 13 260-279 58 58 58 114 114 114 F F F 14 280-299 58 58 58 111 111 111 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 54 54 54 105 105 105 F F F 17 340-359 58 58 58 114 114 114 F F F 18 360-379 58 58 58 114 114 114 T T T 19 380-399 51 51 51 100 100 100 F F F 20 400-419 58 58 58 111 111 111 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 58 58 58 111 111 111 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 58 58 58 111 111 111 F F F 26 520-539 58 58 58 111 111 111 F F F 27 540-559 52 52 52 102 102 102 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 58, Max flush: 114 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 58 58 58 114 114 114 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 58 58 58 114 114 114 F F F 4 080-099 58 58 58 114 114 114 F F F 5 100-119 56 56 56 109 109 109 F F F 6 120-139 58 58 58 114 114 114 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 57 57 57 110 110 110 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 59 59 59 115 115 115 F F F 11 220-239 55 55 55 108 108 108 F F F 12 240-259 58 58 58 114 114 114 F F F 13 260-279 58 58 58 114 114 114 F F F 14 280-299 58 58 58 111 111 111 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 54 54 54 105 105 105 F F F 17 340-359 58 58 58 114 114 114 F F F 18 360-379 59 59 59 115 115 115 F F F 19 380-399 51 51 51 100 100 100 F F F 20 400-419 58 58 58 111 111 111 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 58 58 58 111 111 111 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 50 50 50 98 98 98 F F F 25 500-519 58 58 58 111 111 111 F F F 26 520-539 58 58 58 111 111 111 F F F 27 540-559 52 52 52 102 102 102 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 52 52 52 102 102 102 F F F Max gen: 59, Max flush: 115 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.429301753Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.474473113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.475241423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.475275162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.475288902Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.475303641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} {"msg":"Crucible 14827ecf-b7f6-4ed4-a635-12b73904de9d has session id: 200ae85d-000e-4f41-a82f-baa0937d0b8e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.475953824Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Upstairs opts: Upstairs UUID: 14827ecf-b7f6-4ed4-a635-12b73904de9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.475999253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Crucible stats registered with UUID: 14827ecf-b7f6-4ed4-a635-12b73904de9d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476018113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:53.476031272Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476423392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476451411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} {"msg":"14827ecf-b7f6-4ed4-a635-12b73904de9d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476518189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476535169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"0","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476557099Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"0","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476571058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"1","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476585038Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"1","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476598307Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476612667Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476625557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476646566Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476672476Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64360)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476699025Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37678)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.476725384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:53.476758703Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:53.476788742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:01:53.477034106Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.477174632Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.477563482Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"1","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.477690709Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"0","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.478159186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"1","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.478276673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"0","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.478826659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:01:53.478862968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.478280946Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40255)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.478364784Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.479406326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.479991691Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"2","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[0]R flush_numbers[0..12]: [90, 114, 109, 114, 114, 109, 114, 114, 110, 109, 115, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480607474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[0]R generation[0..12]: [47, 58, 56, 58, 58, 56, 58, 58, 57, 56, 59, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480636014Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480657043Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[1]R flush_numbers[0..12]: [90, 114, 109, 114, 114, 109, 114, 114, 110, 109, 115, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480672093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[1]R generation[0..12]: [47, 58, 56, 58, 58, 56, 58, 58, 57, 56, 59, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480685732Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480699892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[2]R flush_numbers[0..12]: [90, 114, 109, 114, 114, 109, 114, 114, 110, 109, 115, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480713572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[2]R generation[0..12]: [47, 58, 56, 58, 58, 56, 58, 58, 57, 56, 59, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480726911Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480740721Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480753441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.4807665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.48078505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.48079806Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"":"downstairs","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480811129Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480823189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"0","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480836248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"1","":"downstairs","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480849078Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"client":"2","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480870047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480882157Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"14827ecf-b7f6-4ed4-a635-12b73904de9d is now active with session: 200ae85d-000e-4f41-a82f-baa0937d0b8e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480894687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480906417Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432,"session_id":"200ae85d-000e-4f41-a82f-baa0937d0b8e"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480918396Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} {"msg":"Activated sub_volume 14827ecf-b7f6-4ed4-a635-12b73904de9d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:03.480929606Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 540 len 16384 data: 6 5 7 10 02/30 Write block 379 len 8192 data: 6 7 03/30 Read block 20 len 40960 04/30 Read block 243 len 8192 05/30 Read block 358 len 28672 06/30 Write block 31 len 12288 data: 9 8 8 07/30 Read block 485 len 24576 08/30 Read block 586 len 12288 09/30 Read block 267 len 24576 10/30 Write block 359 len 40960 data: 7 8 8 10 9 6 5 6 7 7 11/30 Write block 80 len 20480 data: 6 6 5 8 6 12/30 Write block 114 len 32768 data: 6 4 3 4 5 4 4 3 13/30 Read block 386 len 28672 14/30 Flush 15/30 Write block 115 len 32768 data: 5 4 5 6 5 5 4 3 16/30 Read block 397 len 28672 17/30 Read block 14 len 12288 18/30 Read block 538 len 40960 19/30 Read block 44 len 16384 20/30 Read block 83 len 40960 21/30 Read block 524 len 12288 22/30 Write block 490 len 32768 data: 4 3 2 4 5 6 6 4 23/30 Flush 24/30 Flush 25/30 Write block 590 len 36864 data: 3 3 3 3 3 3 3 2 2 26/30 Read block 446 len 4096 27/30 Read block 264 len 8192 28/30 Read block 84 len 16384 29/30 Read block 81 len 8192 30/30 Write block 66 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:03.502592975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 58 58 58 114 114 114 T T T 4 080-099 60 60 60 116 116 116 F F F 5 100-119 60 60 60 117 117 117 F F F 6 120-139 60 60 60 117 117 117 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 57 57 57 110 110 110 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 59 59 59 115 115 115 F F F 11 220-239 55 55 55 108 108 108 F F F 12 240-259 58 58 58 114 114 114 F F F 13 260-279 58 58 58 114 114 114 F F F 14 280-299 58 58 58 111 111 111 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 54 54 54 105 105 105 F F F 17 340-359 60 60 60 116 116 116 F F F 18 360-379 60 60 60 116 116 116 F F F 19 380-399 60 60 60 116 116 116 F F F 20 400-419 58 58 58 111 111 111 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 58 58 58 111 111 111 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 60 60 60 117 117 117 F F F 25 500-519 58 58 58 111 111 111 F F F 26 520-539 58 58 58 111 111 111 F F F 27 540-559 60 60 60 116 116 116 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 52 52 52 102 102 102 T T T Max gen: 60, Max flush: 117 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 61 61 61 118 118 118 F F F 4 080-099 60 60 60 116 116 116 F F F 5 100-119 60 60 60 117 117 117 F F F 6 120-139 60 60 60 117 117 117 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 57 57 57 110 110 110 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 59 59 59 115 115 115 F F F 11 220-239 55 55 55 108 108 108 F F F 12 240-259 58 58 58 114 114 114 F F F 13 260-279 58 58 58 114 114 114 F F F 14 280-299 58 58 58 111 111 111 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 54 54 54 105 105 105 F F F 17 340-359 60 60 60 116 116 116 F F F 18 360-379 60 60 60 116 116 116 F F F 19 380-399 60 60 60 116 116 116 F F F 20 400-419 58 58 58 111 111 111 F F F 21 420-439 51 51 51 100 100 100 F F F 22 440-459 58 58 58 111 111 111 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 60 60 60 117 117 117 F F F 25 500-519 58 58 58 111 111 111 F F F 26 520-539 58 58 58 111 111 111 F F F 27 540-559 60 60 60 116 116 116 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 61 61 61 118 118 118 F F F Max gen: 61, Max flush: 118 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.725411325Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.769101214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.769902844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.769949122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.769964742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.769979251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} {"msg":"Crucible e054ddce-04e8-44ac-9ea9-0ab67bc45b2c has session id: 272672db-a943-4ae0-85cb-96df1d8c6ce4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.770597525Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Upstairs opts: Upstairs UUID: e054ddce-04e8-44ac-9ea9-0ab67bc45b2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.770630324Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Crucible stats registered with UUID: e054ddce-04e8-44ac-9ea9-0ab67bc45b2c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.770644784Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:07.770657213Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771143901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.77116618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} {"msg":"e054ddce-04e8-44ac-9ea9-0ab67bc45b2c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771244698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771261128Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"0","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771283397Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"0","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771302167Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"1","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771332006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"1","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771358775Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771388514Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771413804Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771440223Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771468142Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"ds_connection connected from Ok(127.0.0.1:61508)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771493851Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:07.771525351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:07.77155708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:07.771587919Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"ds_connection connected from Ok(127.0.0.1:55149)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771655357Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.771806093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.772364219Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"0","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.772391548Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"1","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.77306971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"0","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.773222756Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"1","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.773720133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:07.77383105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.772751452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49707)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.772923598Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.773954051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.774606283Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[0]R flush_numbers[0..12]: [90, 116, 109, 118, 116, 117, 117, 114, 110, 109, 115, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775283456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[0]R generation[0..12]: [47, 60, 56, 61, 60, 60, 60, 58, 57, 56, 59, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775308655Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775324264Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[1]R flush_numbers[0..12]: [90, 116, 109, 118, 116, 117, 117, 114, 110, 109, 115, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775338374Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[1]R generation[0..12]: [47, 60, 56, 61, 60, 60, 60, 58, 57, 56, 59, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775351884Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775365753Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[2]R flush_numbers[0..12]: [90, 116, 109, 118, 116, 117, 117, 114, 110, 109, 115, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775384203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[2]R generation[0..12]: [47, 60, 56, 61, 60, 60, 60, 58, 57, 56, 59, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775397842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775411562Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775424252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775437132Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775450011Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775462541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"":"downstairs","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.77547538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.77548715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"0","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.77550007Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"1","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775518829Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"client":"2","":"downstairs","session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775531869Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775543769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"e054ddce-04e8-44ac-9ea9-0ab67bc45b2c is now active with session: 272672db-a943-4ae0-85cb-96df1d8c6ce4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775556168Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775567588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440,"session_id":"272672db-a943-4ae0-85cb-96df1d8c6ce4"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775579418Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} {"msg":"Activated sub_volume e054ddce-04e8-44ac-9ea9-0ab67bc45b2c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:17.775592107Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 12288 data: 4 2 4 02/30 Write block 354 len 36864 data: 4 4 4 4 5 8 9 9 11 03/30 Read block 378 len 20480 04/30 Read block 578 len 36864 05/30 Read block 158 len 12288 06/30 Read block 32 len 40960 07/30 Write block 346 len 32768 data: 7 6 7 8 6 7 5 4 08/30 Write block 282 len 24576 data: 5 5 5 4 3 3 09/30 Read block 506 len 28672 10/30 Write block 416 len 20480 data: 8 7 7 9 9 11/30 Flush 12/30 Write block 521 len 32768 data: 4 4 5 4 4 4 5 5 13/30 Read block 281 len 40960 14/30 Flush 15/30 Write block 259 len 40960 data: 6 6 8 9 6 6 6 6 7 7 16/30 Write block 117 len 32768 data: 6 7 6 6 5 4 4 4 17/30 Flush 18/30 Write block 532 len 8192 data: 7 6 19/30 Read block 103 len 28672 20/30 Write block 563 len 12288 data: 6 8 8 21/30 Write block 103 len 36864 data: 11 9 10 7 6 5 6 6 5 22/30 Write block 121 len 36864 data: 6 5 5 5 5 2 3 3 3 23/30 Read block 560 len 4096 24/30 Write block 218 len 40960 data: 4 3 2 3 4 4 7 6 8 9 25/30 Read block 80 len 24576 26/30 Write block 341 len 32768 data: 7 8 8 9 9 8 7 8 27/30 Read block 363 len 20480 28/30 Write block 490 len 20480 data: 5 4 3 5 6 29/30 Write block 224 len 16384 data: 8 7 9 10 30/30 Read block 513 len 24576 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 2 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 12 11 13 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:17.798819145Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 61 61 61 118 118 118 F F F 4 080-099 60 60 60 116 116 116 F F F 5 100-119 62 62 62 121 121 121 T T T 6 120-139 62 62 62 121 121 121 T T T 7 140-159 58 58 58 114 114 114 F F F 8 160-179 57 57 57 110 110 110 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 59 59 59 115 115 115 T T T 11 220-239 55 55 55 108 108 108 T T T 12 240-259 62 62 62 121 121 121 F F F 13 260-279 62 62 62 121 121 121 F F F 14 280-299 62 62 62 119 119 119 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 62 62 62 119 119 119 F F F 17 340-359 62 62 62 119 119 119 T T T 18 360-379 62 62 62 119 119 119 F F F 19 380-399 60 60 60 116 116 116 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 58 58 58 111 111 111 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 60 60 60 117 117 117 T T T 25 500-519 58 58 58 111 111 111 F F F 26 520-539 62 62 62 120 120 120 T T T 27 540-559 60 60 60 116 116 116 F F F 28 560-579 54 54 54 105 105 105 T T T 29 580-599 61 61 61 118 118 118 F F F Max gen: 62, Max flush: 121 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 90 90 90 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 61 61 61 118 118 118 F F F 4 080-099 60 60 60 116 116 116 F F F 5 100-119 63 63 63 122 122 122 F F F 6 120-139 63 63 63 122 122 122 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 57 57 57 110 110 110 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 63 63 63 122 122 122 F F F 12 240-259 62 62 62 121 121 121 F F F 13 260-279 62 62 62 121 121 121 F F F 14 280-299 62 62 62 119 119 119 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 62 62 62 119 119 119 F F F 17 340-359 63 63 63 122 122 122 F F F 18 360-379 62 62 62 119 119 119 F F F 19 380-399 60 60 60 116 116 116 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 58 58 58 111 111 111 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 63 63 63 122 122 122 F F F 25 500-519 58 58 58 111 111 111 F F F 26 520-539 63 63 63 122 122 122 F F F 27 540-559 60 60 60 116 116 116 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 61 61 61 118 118 118 F F F Max gen: 63, Max flush: 122 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.054553669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.098138091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.0989088Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.098957149Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.098978578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.098994448Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} {"msg":"Crucible 01f3d5d4-363d-4828-9480-954104450a29 has session id: f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.099553993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Upstairs opts: Upstairs UUID: 01f3d5d4-363d-4828-9480-954104450a29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.099600332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Crucible stats registered with UUID: 01f3d5d4-363d-4828-9480-954104450a29","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.099626961Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:22.099648951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.10006736Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100104189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} {"msg":"01f3d5d4-363d-4828-9480-954104450a29 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100128528Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100143098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"0","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100165997Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"0","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100179537Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100193497Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100206706Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"2","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100220246Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"2","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100238855Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100253095Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100266544Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34001)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100355122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:33525)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100419Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:22.10045116Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:22.100487449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:22.100604046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.100680154Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.101144651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"0","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.101175391Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"2","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.101837243Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"0","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.101989429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"2","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.102430218Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:22.102593333Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.102896478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56392)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.102984926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.103995739Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.104631402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[0]R flush_numbers[0..12]: [90, 116, 109, 118, 116, 122, 122, 114, 110, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105256376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[0]R generation[0..12]: [47, 60, 56, 61, 60, 63, 63, 58, 57, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105286055Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105303724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[1]R flush_numbers[0..12]: [90, 116, 109, 118, 116, 122, 122, 114, 110, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105318744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[1]R generation[0..12]: [47, 60, 56, 61, 60, 63, 63, 58, 57, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105332714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105347193Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[2]R flush_numbers[0..12]: [90, 116, 109, 118, 116, 122, 122, 114, 110, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105361063Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[2]R generation[0..12]: [47, 60, 56, 61, 60, 63, 63, 58, 57, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105374993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105393652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105407222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105420612Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105433541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105446621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"":"downstairs","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.1054604Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.10547278Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"0","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.10548648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"1","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105499859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"client":"2","":"downstairs","session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105513769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105526149Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"01f3d5d4-363d-4828-9480-954104450a29 is now active with session: f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105539148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105551168Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448,"session_id":"f46257cf-0eb0-4a2f-bb9d-17a19fe3b5eb"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:32.105563618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} {"msg":"Activated sub_volume 01f3d5d4-363d-4828-9480-954104450a29","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-19T17:02:32.105574817Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 119 len 8192 data: 7 7 02/30 Write block 444 len 16384 data: 4 5 6 5 03/30 Write block 82 len 4096 data: 6 04/30 Write block 390 len 32768 data: 3 7 7 7 9 9 7 8 05/30 Flush 06/30 Write block 10 len 4096 data: 3 07/30 Write block 246 len 16384 data: 5 9 9 8 08/30 Read block 269 len 4096 09/30 Write block 289 len 8192 data: 2 2 10/30 Read block 540 len 32768 11/30 Write block 331 len 12288 data: 6 6 8 12/30 Write block 588 len 24576 data: 3 2 4 4 4 4 13/30 Write block 354 len 24576 data: 5 5 5 5 6 9 14/30 Read block 241 len 20480 15/30 Write block 257 len 4096 data: 3 16/30 Write block 342 len 28672 data: 9 9 10 10 9 8 9 17/30 Flush 18/30 Write block 78 len 8192 data: 6 6 19/30 Write block 173 len 12288 data: 7 6 7 20/30 Read block 328 len 28672 21/30 Read block 169 len 36864 22/30 Read block 378 len 28672 23/30 Flush 24/30 Flush 25/30 Write block 3 len 12288 data: 2 2 2 26/30 Read block 204 len 40960 27/30 Write block 374 len 24576 data: 6 2 3 3 4 7 28/30 Read block 378 len 20480 29/30 Write block 377 len 32768 data: 4 5 8 8 7 6 6 6 30/30 Write block 510 len 12288 data: 7 9 7 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:32.124700924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 124 124 124 T T T 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 64 64 64 123 123 123 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 64 64 64 125 125 125 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 63 63 63 122 122 122 F F F 12 240-259 64 64 64 124 124 124 F F F 13 260-279 62 62 62 121 121 121 F F F 14 280-299 64 64 64 124 124 124 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 64 64 64 124 124 124 F F F 17 340-359 64 64 64 124 124 124 F F F 18 360-379 62 62 62 119 119 119 T T T 19 380-399 64 64 64 123 123 123 T T T 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 64 64 64 123 123 123 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 63 63 63 122 122 122 F F F 25 500-519 58 58 58 111 111 111 T T T 26 520-539 63 63 63 122 122 122 F F F 27 540-559 60 60 60 116 116 116 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 64 64 64 124 124 124 F F F Max gen: 64, Max flush: 125 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 126 126 126 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 64 64 64 123 123 123 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 64 64 64 125 125 125 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 63 63 63 122 122 122 F F F 12 240-259 64 64 64 124 124 124 F F F 13 260-279 62 62 62 121 121 121 F F F 14 280-299 64 64 64 124 124 124 F F F 15 300-319 57 57 57 110 110 110 F F F 16 320-339 64 64 64 124 124 124 F F F 17 340-359 64 64 64 124 124 124 F F F 18 360-379 65 65 65 126 126 126 F F F 19 380-399 65 65 65 126 126 126 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 64 64 64 123 123 123 F F F 23 460-479 58 58 58 111 111 111 F F F 24 480-499 63 63 63 122 122 122 F F F 25 500-519 65 65 65 126 126 126 F F F 26 520-539 63 63 63 122 122 122 F F F 27 540-559 60 60 60 116 116 116 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 64 64 64 124 124 124 F F F Max gen: 65, Max flush: 126 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.3587245Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:36.400725823Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.401102633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.432110406Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.432809988Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.432845847Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.432859976Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.432874296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} {"msg":"Crucible 2b613873-9210-4986-af58-790b90af8e92 has session id: 00b74a4b-0858-4175-bd27-f660d2b36513","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.43347858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Upstairs opts: Upstairs UUID: 2b613873-9210-4986-af58-790b90af8e92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.433522479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Crucible stats registered with UUID: 2b613873-9210-4986-af58-790b90af8e92","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.433546388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:36.433567088Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.433929268Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.433955507Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} {"msg":"2b613873-9210-4986-af58-790b90af8e92 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.433997126Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434012596Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"0","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434034656Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"0","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434054735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"1","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434081534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"1","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434106343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"2","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434134253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"2","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434161192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434193651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434220351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"ds_connection connected from Ok(127.0.0.1:64626)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.43424486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"ds_connection connected from Ok(127.0.0.1:39436)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434269019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"ds_connection connected from Ok(127.0.0.1:37751)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434424835Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434948181Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"1","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.434979951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"2","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.435294292Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"0","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.435736351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"1","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.43576451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"2","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.435912706Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"0","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436358154Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436453862Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[0]R flush_numbers[0..12]: [126, 116, 109, 125, 123, 123, 123, 114, 125, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.43652391Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[0]R generation[0..12]: [65, 60, 56, 64, 64, 64, 64, 58, 64, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436544029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436562309Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[1]R flush_numbers[0..12]: [126, 116, 109, 125, 123, 123, 123, 114, 125, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436576349Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[1]R generation[0..12]: [65, 60, 56, 64, 64, 64, 64, 58, 64, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436589758Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436603368Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[2]R flush_numbers[0..12]: [126, 116, 109, 125, 123, 123, 123, 114, 125, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436616528Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[2]R generation[0..12]: [65, 60, 56, 64, 64, 64, 64, 58, 64, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436629467Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436642917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436655626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436668556Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436681076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436693745Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"":"downstairs","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436706375Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436718125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"0","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:02:36.436731054Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"1","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436753014Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"client":"2","":"downstairs","session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436766233Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436778083Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"2b613873-9210-4986-af58-790b90af8e92 is now active with session: 00b74a4b-0858-4175-bd27-f660d2b36513","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436790473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436802472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456,"session_id":"00b74a4b-0858-4175-bd27-f660d2b36513"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436814492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} {"msg":"Activated sub_volume 2b613873-9210-4986-af58-790b90af8e92","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:36.436825392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 346 len 20480 data: 10 9 10 9 7 03/30 Read block 7 len 8192 04/30 Read block 456 len 28672 05/30 Read block 343 len 24576 06/30 Write block 139 len 36864 data: 7 6 6 4 4 3 5 4 4 07/30 Read block 45 len 12288 08/30 Read block 217 len 36864 09/30 Write block 373 len 12288 data: 7 7 3 10/30 Read block 504 len 4096 11/30 Read block 361 len 20480 12/30 Read block 30 len 32768 13/30 Read block 68 len 20480 14/30 Read block 340 len 16384 15/30 Read block 185 len 12288 16/30 Read block 467 len 40960 17/30 Read block 253 len 32768 18/30 Write block 510 len 28672 data: 8 10 8 7 9 5 5 19/30 Read block 414 len 8192 20/30 Read block 543 len 28672 21/30 Read block 152 len 4096 22/30 Flush 23/30 Write block 321 len 8192 data: 5 4 24/30 Read block 484 len 20480 25/30 Write block 246 len 8192 data: 6 10 26/30 Read block 559 len 28672 27/30 Write block 463 len 4096 data: 5 28/30 Read block 220 len 40960 29/30 Write block 316 len 28672 data: 8 7 5 6 6 6 5 30/30 Write block 133 len 16384 data: 5 5 6 6 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:36.457036559Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 126 126 126 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 66 66 66 128 128 128 T T T 7 140-159 66 66 66 128 128 128 F F F 8 160-179 64 64 64 125 125 125 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 63 63 63 122 122 122 F F F 12 240-259 64 64 64 124 124 124 T T T 13 260-279 62 62 62 121 121 121 F F F 14 280-299 64 64 64 124 124 124 F F F 15 300-319 57 57 57 110 110 110 T T T 16 320-339 64 64 64 124 124 124 T T T 17 340-359 66 66 66 128 128 128 F F F 18 360-379 66 66 66 128 128 128 F F F 19 380-399 65 65 65 126 126 126 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 64 64 64 123 123 123 F F F 23 460-479 58 58 58 111 111 111 T T T 24 480-499 63 63 63 122 122 122 F F F 25 500-519 66 66 66 128 128 128 F F F 26 520-539 63 63 63 122 122 122 F F F 27 540-559 60 60 60 116 116 116 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 64 64 64 124 124 124 F F F Max gen: 66, Max flush: 128 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 126 126 126 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 67 67 67 129 129 129 F F F 7 140-159 66 66 66 128 128 128 F F F 8 160-179 64 64 64 125 125 125 F F F 9 180-199 56 56 56 109 109 109 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 63 63 63 122 122 122 F F F 12 240-259 67 67 67 129 129 129 F F F 13 260-279 62 62 62 121 121 121 F F F 14 280-299 64 64 64 124 124 124 F F F 15 300-319 67 67 67 129 129 129 F F F 16 320-339 67 67 67 129 129 129 F F F 17 340-359 66 66 66 128 128 128 F F F 18 360-379 66 66 66 128 128 128 F F F 19 380-399 65 65 65 126 126 126 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 64 64 64 123 123 123 F F F 23 460-479 67 67 67 129 129 129 F F F 24 480-499 63 63 63 122 122 122 F F F 25 500-519 66 66 66 128 128 128 F F F 26 520-539 63 63 63 122 122 122 F F F 27 540-559 60 60 60 116 116 116 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 64 64 64 124 124 124 F F F Max gen: 67, Max flush: 129 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.731047111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.775140689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.77587585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.775912169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.775925659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.775939688Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} {"msg":"Crucible 95dd0fc4-574b-4191-92f5-aeced1899e28 has session id: 7f9cccad-f395-4148-aa3a-445344dbec31","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.776550622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Upstairs opts: Upstairs UUID: 95dd0fc4-574b-4191-92f5-aeced1899e28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.776612211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Crucible stats registered with UUID: 95dd0fc4-574b-4191-92f5-aeced1899e28","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.77663917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:40.776660319Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777186645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777212124Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} {"msg":"95dd0fc4-574b-4191-92f5-aeced1899e28 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777285342Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777301962Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"0","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777324941Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"0","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.77734598Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.77737231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777396559Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"2","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777425678Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"2","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777453018Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777477227Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777500107Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"ds_connection connected from Ok(127.0.0.1:48245)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777522006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"ds_connection connected from Ok(127.0.0.1:54566)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777542975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:40.777800608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:40.777847737Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:40.777879777Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.777977484Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.778315735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"0","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.778343314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"2","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.778960618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"2","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.779114474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"0","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.779493894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:40.779609001Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.779301242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"ds_connection connected from Ok(127.0.0.1:60133)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.77937651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.780544649Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781135814Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[0]R flush_numbers[0..12]: [126, 116, 109, 125, 123, 123, 129, 128, 125, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781759927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[0]R generation[0..12]: [65, 60, 56, 64, 64, 64, 67, 66, 64, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781788926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781805096Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[1]R flush_numbers[0..12]: [126, 116, 109, 125, 123, 123, 129, 128, 125, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781819066Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[1]R generation[0..12]: [65, 60, 56, 64, 64, 64, 67, 66, 64, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781832405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781851165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[2]R flush_numbers[0..12]: [126, 116, 109, 125, 123, 123, 129, 128, 125, 109, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781865124Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[2]R generation[0..12]: [65, 60, 56, 64, 64, 64, 67, 66, 64, 56, 63, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781878454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781892004Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:02:50.781904563Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781919473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781932463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781945442Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"":"downstairs","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781958412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781969972Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"0","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781982681Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"1","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.781995251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"client":"2","":"downstairs","session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.782008101Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.78202003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"95dd0fc4-574b-4191-92f5-aeced1899e28 is now active with session: 7f9cccad-f395-4148-aa3a-445344dbec31","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.78203257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.782051329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464,"session_id":"7f9cccad-f395-4148-aa3a-445344dbec31"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.782063429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} {"msg":"Activated sub_volume 95dd0fc4-574b-4191-92f5-aeced1899e28","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:50.782074359Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 456 len 32768 data: 5 5 6 5 4 5 5 6 04/30 Read block 546 len 40960 05/30 Read block 165 len 12288 06/30 Write block 540 len 16384 data: 7 6 8 11 07/30 Write block 159 len 8192 data: 5 4 08/30 Write block 328 len 24576 data: 4 4 3 7 7 9 09/30 Write block 4 len 28672 data: 3 3 4 4 3 3 4 10/30 Write block 162 len 12288 data: 5 4 6 11/30 Read block 126 len 32768 12/30 Flush 13/30 Write block 240 len 40960 data: 5 6 7 8 4 5 7 11 10 9 14/30 Read block 496 len 32768 15/30 Read block 177 len 36864 16/30 Flush 17/30 Write block 143 len 36864 data: 5 4 6 5 5 4 5 5 6 18/30 Read block 506 len 28672 19/30 Read block 175 len 32768 20/30 Read block 487 len 32768 21/30 Write block 568 len 12288 data: 9 7 6 22/30 Read block 129 len 40960 23/30 Write block 202 len 4096 data: 8 24/30 Read block 97 len 20480 25/30 Read block 467 len 24576 26/30 Read block 453 len 20480 27/30 Read block 396 len 16384 28/30 Write block 193 len 16384 data: 4 4 4 5 29/30 Read block 278 len 28672 30/30 Write block 528 len 8192 data: 6 5 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 9 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 7 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 13 12 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:50.80291716Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 67 67 67 129 129 129 F F F 7 140-159 68 68 68 132 132 132 T T T 8 160-179 68 68 68 132 132 132 F F F 9 180-199 56 56 56 109 109 109 T T T 10 200-219 63 63 63 122 122 122 T T T 11 220-239 63 63 63 122 122 122 F F F 12 240-259 68 68 68 133 133 133 F F F 13 260-279 62 62 62 121 121 121 F F F 14 280-299 64 64 64 124 124 124 F F F 15 300-319 67 67 67 129 129 129 F F F 16 320-339 68 68 68 132 132 132 F F F 17 340-359 66 66 66 128 128 128 F F F 18 360-379 66 66 66 128 128 128 F F F 19 380-399 65 65 65 126 126 126 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 68 68 68 132 132 132 F F F 23 460-479 68 68 68 132 132 132 F F F 24 480-499 63 63 63 122 122 122 F F F 25 500-519 66 66 66 128 128 128 F F F 26 520-539 63 63 63 122 122 122 T T T 27 540-559 68 68 68 132 132 132 F F F 28 560-579 63 63 63 122 122 122 T T T 29 580-599 64 64 64 124 124 124 F F F Max gen: 68, Max flush: 133 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 F F F 1 020-039 60 60 60 116 116 116 F F F 2 040-059 56 56 56 109 109 109 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 67 67 67 129 129 129 F F F 7 140-159 69 69 69 134 134 134 F F F 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 F F F 10 200-219 69 69 69 134 134 134 F F F 11 220-239 63 63 63 122 122 122 F F F 12 240-259 68 68 68 133 133 133 F F F 13 260-279 62 62 62 121 121 121 F F F 14 280-299 64 64 64 124 124 124 F F F 15 300-319 67 67 67 129 129 129 F F F 16 320-339 68 68 68 132 132 132 F F F 17 340-359 66 66 66 128 128 128 F F F 18 360-379 66 66 66 128 128 128 F F F 19 380-399 65 65 65 126 126 126 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 68 68 68 132 132 132 F F F 23 460-479 68 68 68 132 132 132 F F F 24 480-499 63 63 63 122 122 122 F F F 25 500-519 66 66 66 128 128 128 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 68 68 68 132 132 132 F F F 28 560-579 69 69 69 134 134 134 F F F 29 580-599 64 64 64 124 124 124 F F F Max gen: 69, Max flush: 134 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.046208442Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.090560923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.091408741Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.09145296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.091473139Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.091488189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} {"msg":"Crucible b0895384-361a-4b67-8129-d6fe12184dd3 has session id: 78378368-88d5-49b9-b083-ae3e3cc6d3d9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092092113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Upstairs opts: Upstairs UUID: b0895384-361a-4b67-8129-d6fe12184dd3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092127902Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Crucible stats registered with UUID: b0895384-361a-4b67-8129-d6fe12184dd3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092143781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:55.092156791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092523302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092557021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} {"msg":"b0895384-361a-4b67-8129-d6fe12184dd3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092612429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092643818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"0","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092683277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"0","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092713986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"1","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092821203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"1","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092862202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"2","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092891692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"2","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092914771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.09293888Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.09296527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:49503)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.092991409Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64397)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.093015629Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37146)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.093040508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.093434767Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"2","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.093463147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"0","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.094161818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"0","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.094249896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"2","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.09447147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"1","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.094747363Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.094913399Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095107933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"1","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[0]R flush_numbers[0..12]: [132, 116, 109, 125, 123, 123, 129, 134, 132, 134, 134, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.09562676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[0]R generation[0..12]: [68, 60, 56, 64, 64, 64, 67, 69, 68, 69, 69, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095654819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095670268Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[1]R flush_numbers[0..12]: [132, 116, 109, 125, 123, 123, 129, 134, 132, 134, 134, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095683998Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[1]R generation[0..12]: [68, 60, 56, 64, 64, 64, 67, 69, 68, 69, 69, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095697298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095711127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[2]R flush_numbers[0..12]: [132, 116, 109, 125, 123, 123, 129, 134, 132, 134, 134, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095724987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[2]R generation[0..12]: [68, 60, 56, 64, 64, 64, 67, 69, 68, 69, 69, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095738287Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095751986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095764596Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095777346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095790105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095802835Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095829924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095841614Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"0","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095854724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"1","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095867513Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"client":"2","":"downstairs","session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095880443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095892423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"b0895384-361a-4b67-8129-d6fe12184dd3 is now active with session: 78378368-88d5-49b9-b083-ae3e3cc6d3d9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095904922Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095916652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472,"session_id":"78378368-88d5-49b9-b083-ae3e3cc6d3d9"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095928602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} {"msg":"Activated sub_volume b0895384-361a-4b67-8129-d6fe12184dd3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:02:55.095939612Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 403 len 4096 data: 7 02/30 Read block 91 len 8192 03/30 Write block 261 len 12288 data: 9 10 7 04/30 Read block 59 len 12288 05/30 Read block 203 len 12288 06/30 Flush 07/30 Flush 08/30 Write block 340 len 8192 data: 8 8 09/30 Read block 456 len 4096 10/30 Write block 573 len 32768 data: 4 3 4 4 4 5 4 4 11/30 Write block 273 len 36864 data: 7 8 7 6 6 6 6 5 5 12/30 Write block 317 len 12288 data: 8 6 7 13/30 Read block 403 len 32768 14/30 Write block 594 len 12288 data: 4 4 4 15/30 Write block 511 len 24576 data: 11 9 8 10 6 6 16/30 Write block 206 len 16384 data: 5 5 5 6 17/30 Write block 273 len 12288 data: 8 9 8 18/30 Read block 260 len 12288 19/30 Write block 54 len 36864 data: 8 6 5 6 3 4 4 4 6 20/30 Write block 245 len 4096 data: 6 21/30 Write block 58 len 32768 data: 4 5 5 5 7 7 6 6 22/30 Read block 289 len 40960 23/30 Write block 478 len 36864 data: 5 6 8 7 5 5 5 5 6 24/30 Write block 293 len 32768 data: 5 6 5 4 3 3 3 5 25/30 Write block 394 len 32768 data: 10 10 8 9 10 8 9 9 26/30 Write block 141 len 28672 data: 7 5 6 5 7 6 6 27/30 Write block 13 len 32768 data: 4 4 4 3 4 6 6 5 28/30 Write block 494 len 20480 data: 7 7 7 5 4 29/30 Read block 39 len 16384 30/30 Read block 519 len 16384 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:02:55.114872423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 68 132 132 132 T T T 1 020-039 60 60 60 116 116 116 T T T 2 040-059 56 56 56 109 109 109 T T T 3 060-079 64 64 64 125 125 125 T T T 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 67 67 67 129 129 129 F F F 7 140-159 69 69 69 134 134 134 T T T 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 F F F 10 200-219 69 69 69 134 134 134 T T T 11 220-239 63 63 63 122 122 122 F F F 12 240-259 68 68 68 133 133 133 T T T 13 260-279 70 70 70 135 135 135 T T T 14 280-299 64 64 64 124 124 124 T T T 15 300-319 67 67 67 129 129 129 T T T 16 320-339 68 68 68 132 132 132 F F F 17 340-359 66 66 66 128 128 128 T T T 18 360-379 66 66 66 128 128 128 F F F 19 380-399 65 65 65 126 126 126 T T T 20 400-419 70 70 70 135 135 135 T T T 21 420-439 62 62 62 119 119 119 F F F 22 440-459 68 68 68 132 132 132 F F F 23 460-479 68 68 68 132 132 132 T T T 24 480-499 63 63 63 122 122 122 T T T 25 500-519 66 66 66 128 128 128 T T T 26 520-539 69 69 69 134 134 134 F F F 27 540-559 68 68 68 132 132 132 F F F 28 560-579 69 69 69 134 134 134 T T T 29 580-599 64 64 64 124 124 124 T T T Max gen: 70, Max flush: 135 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 136 136 136 F F F 1 020-039 71 71 71 136 136 136 F F F 2 040-059 71 71 71 136 136 136 F F F 3 060-079 71 71 71 136 136 136 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 67 67 67 129 129 129 F F F 7 140-159 71 71 71 136 136 136 F F F 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 63 63 63 122 122 122 F F F 12 240-259 71 71 71 136 136 136 F F F 13 260-279 71 71 71 136 136 136 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 71 71 71 136 136 136 F F F 16 320-339 68 68 68 132 132 132 F F F 17 340-359 71 71 71 136 136 136 F F F 18 360-379 66 66 66 128 128 128 F F F 19 380-399 71 71 71 136 136 136 F F F 20 400-419 71 71 71 136 136 136 F F F 21 420-439 62 62 62 119 119 119 F F F 22 440-459 68 68 68 132 132 132 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 71 71 71 136 136 136 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 68 68 68 132 132 132 F F F 28 560-579 71 71 71 136 136 136 F F F 29 580-599 71 71 71 136 136 136 F F F Max gen: 71, Max flush: 136 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.419348175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:09.462371732Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.462789781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.494458217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.49509676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.495130419Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.495144198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.495158248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} {"msg":"Crucible 46ba5df8-3562-4797-8a2d-71707ffd0cfb has session id: 84abdc25-64f5-427c-bd52-fce0df04af9d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.495798391Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Upstairs opts: Upstairs UUID: 46ba5df8-3562-4797-8a2d-71707ffd0cfb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.49584051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Crucible stats registered with UUID: 46ba5df8-3562-4797-8a2d-71707ffd0cfb","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.49586351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:09.495883469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496291538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496318488Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} {"msg":"46ba5df8-3562-4797-8a2d-71707ffd0cfb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496379066Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496396465Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"0","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496422715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"0","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496437595Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"1","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496452214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"1","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496465344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"2","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496479093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"2","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496492173Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496505653Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496523302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63810)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496639769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33988)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496759346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36550)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.496872143Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.497450828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"1","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.497486427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"2","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.497617723Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"0","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498169199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"1","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498212048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"2","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498244837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"0","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498852021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.49887669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[0]R flush_numbers[0..12]: [136, 136, 136, 136, 123, 123, 129, 136, 132, 134, 136, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498948488Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[0]R generation[0..12]: [71, 71, 71, 71, 64, 64, 67, 71, 68, 69, 71, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498967778Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498985567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[1]R flush_numbers[0..12]: [136, 136, 136, 136, 123, 123, 129, 136, 132, 134, 136, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.498999887Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[1]R generation[0..12]: [71, 71, 71, 71, 64, 64, 67, 71, 68, 69, 71, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499013607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499027516Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[2]R flush_numbers[0..12]: [136, 136, 136, 136, 123, 123, 129, 136, 132, 134, 136, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499041156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[2]R generation[0..12]: [71, 71, 71, 71, 64, 64, 67, 71, 68, 69, 71, 63]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499054366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499067975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499080535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499093254Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499109964Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:03:09.499133273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"":"downstairs","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499153633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499165963Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"0","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499179072Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"1","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499195102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"client":"2","":"downstairs","session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499221971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499238131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"46ba5df8-3562-4797-8a2d-71707ffd0cfb is now active with session: 84abdc25-64f5-427c-bd52-fce0df04af9d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.49925089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.49927065Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480,"session_id":"84abdc25-64f5-427c-bd52-fce0df04af9d"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499291599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} {"msg":"Activated sub_volume 46ba5df8-3562-4797-8a2d-71707ffd0cfb","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:09.499303699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 302 len 16384 data: 4 4 4 5 02/30 Flush 03/30 Flush 04/30 Write block 435 len 16384 data: 5 5 5 5 05/30 Write block 544 len 32768 data: 10 10 9 13 12 11 11 11 06/30 Write block 235 len 28672 data: 6 6 6 5 5 6 7 07/30 Read block 270 len 32768 08/30 Read block 293 len 12288 09/30 Read block 356 len 4096 10/30 Read block 389 len 32768 11/30 Write block 575 len 32768 data: 5 5 5 6 5 5 4 4 12/30 Flush 13/30 Write block 577 len 40960 data: 6 7 6 6 5 5 5 4 4 3 14/30 Write block 334 len 36864 data: 7 7 8 8 10 8 9 9 10 15/30 Read block 58 len 12288 16/30 Write block 323 len 32768 data: 5 5 3 5 4 5 5 4 17/30 Write block 261 len 40960 data: 10 11 8 7 7 7 8 8 6 6 18/30 Write block 374 len 24576 data: 8 4 4 5 6 9 19/30 Read block 362 len 32768 20/30 Read block 189 len 40960 21/30 Read block 186 len 8192 22/30 Read block 224 len 16384 23/30 Read block 96 len 8192 24/30 Write block 371 len 32768 data: 6 7 8 9 5 5 6 7 25/30 Write block 450 len 12288 data: 5 7 7 26/30 Write block 31 len 8192 data: 10 9 27/30 Write block 68 len 32768 data: 5 5 4 4 2 2 5 7 28/30 Write block 275 len 12288 data: 9 7 7 29/30 Write block 402 len 28672 data: 7 8 5 4 4 5 5 30/30 Read block 282 len 16384 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 10 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 17 18 16 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:09.519289002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 136 136 136 F F F 1 020-039 71 71 71 136 136 136 T T T 2 040-059 71 71 71 136 136 136 F F F 3 060-079 71 71 71 136 136 136 T T T 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 67 67 67 129 129 129 F F F 7 140-159 71 71 71 136 136 136 F F F 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 72 72 72 139 139 139 F F F 12 240-259 72 72 72 139 139 139 F F F 13 260-279 71 71 71 136 136 136 T T T 14 280-299 71 71 71 136 136 136 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 68 68 68 132 132 132 T T T 17 340-359 71 71 71 136 136 136 T T T 18 360-379 66 66 66 128 128 128 T T T 19 380-399 71 71 71 136 136 136 F F F 20 400-419 71 71 71 136 136 136 T T T 21 420-439 72 72 72 139 139 139 F F F 22 440-459 68 68 68 132 132 132 T T T 23 460-479 71 71 71 136 136 136 F F F 24 480-499 71 71 71 136 136 136 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 72 72 72 139 139 139 F F F 28 560-579 72 72 72 139 139 139 T T T 29 580-599 72 72 72 139 139 139 T T T Max gen: 72, Max flush: 139 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 136 136 136 F F F 1 020-039 73 73 73 140 140 140 F F F 2 040-059 71 71 71 136 136 136 F F F 3 060-079 73 73 73 140 140 140 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 67 67 67 129 129 129 F F F 7 140-159 71 71 71 136 136 136 F F F 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 72 72 72 139 139 139 F F F 12 240-259 72 72 72 139 139 139 F F F 13 260-279 73 73 73 140 140 140 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 73 73 73 140 140 140 F F F 17 340-359 73 73 73 140 140 140 F F F 18 360-379 73 73 73 140 140 140 F F F 19 380-399 71 71 71 136 136 136 F F F 20 400-419 73 73 73 140 140 140 F F F 21 420-439 72 72 72 139 139 139 F F F 22 440-459 73 73 73 140 140 140 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 71 71 71 136 136 136 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 72 72 72 139 139 139 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 73 73 73 140 140 140 F F F Max gen: 73, Max flush: 140 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.810015303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.853147257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.853585565Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.885525793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.886384021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.88641637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.8864297Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.886449289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} {"msg":"Crucible 994b3020-4856-4f6e-b458-c15530c65219 has session id: 4fa371ce-b961-48e6-9164-49f3a30802b4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.886981895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Upstairs opts: Upstairs UUID: 994b3020-4856-4f6e-b458-c15530c65219, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887097272Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Crucible stats registered with UUID: 994b3020-4856-4f6e-b458-c15530c65219","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887124691Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.887138851Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887586329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887609729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} {"msg":"994b3020-4856-4f6e-b458-c15530c65219 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887670767Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887688547Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"0","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887711346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"0","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887724986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"1","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887739675Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"1","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887817723Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"2","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887859542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"2","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887889171Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887915771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.88794437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43951)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887972979Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:39131)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.887999718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41924)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.888040227Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.888666811Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"2","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.8886952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"1","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.888936924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"0","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.889353973Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"1","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.889382892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"2","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.889695984Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"0","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890132022Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890172061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[0]R flush_numbers[0..12]: [136, 140, 136, 140, 123, 123, 129, 136, 132, 134, 136, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890307858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[0]R generation[0..12]: [71, 73, 71, 73, 64, 64, 67, 71, 68, 69, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890328447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890343327Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[1]R flush_numbers[0..12]: [136, 140, 136, 140, 123, 123, 129, 136, 132, 134, 136, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890357186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[1]R generation[0..12]: [71, 73, 71, 73, 64, 64, 67, 71, 68, 69, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890370526Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890384436Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[2]R flush_numbers[0..12]: [136, 140, 136, 140, 123, 123, 129, 136, 132, 134, 136, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890401635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[2]R generation[0..12]: [71, 73, 71, 73, 64, 64, 67, 71, 68, 69, 71, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890415505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890429064Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890441654Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890454264Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890466743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890486563Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"":"downstairs","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890512172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890541492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"0","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890563111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"1","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890576631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"client":"2","":"downstairs","session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.89059158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890612479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"994b3020-4856-4f6e-b458-c15530c65219 is now active with session: 4fa371ce-b961-48e6-9164-49f3a30802b4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890636169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890657408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890679308Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} {"msg":"Activated sub_volume 994b3020-4856-4f6e-b458-c15530c65219","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:13.890702157Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 14 len 36864 data: 5 5 4 5 7 7 6 6 7 03/30 Write block 354 len 8192 data: 6 6 04/30 Read block 426 len 32768 05/30 Write block 150 len 4096 data: 6 06/30 Flush 07/30 Write block 495 len 20480 data: 8 8 6 5 2 08/30 Read block 24 len 36864 09/30 Write block 323 len 24576 data: 6 6 4 6 5 6 10/30 Read block 261 len 40960 11/30 Read block 204 len 36864 12/30 Flush 13/30 Flush 14/30 Write block 132 len 12288 data: 5 6 6 15/30 Read block 579 len 40960 16/30 Write block 404 len 20480 data: 6 5 5 6 6 17/30 Flush 18/30 Write block 9 len 36864 data: 4 5 4 4 5 6 6 5 6 19/30 Read block 284 len 28672 20/30 Write block 223 len 24576 data: 5 9 8 10 11 8 21/30 Flush 22/30 Write block 82 len 32768 data: 7 9 7 6 7 7 8 10 23/30 Read block 179 len 28672 24/30 Flush 25/30 Write block 127 len 12288 data: 4 4 4 26/30 Read block 54 len 32768 27/30 Write block 140 len 32768 data: 7 8 6 7 6 8 7 7 28/30 Write block 92 len 32768 data: 7 10 9 8 10 10 9 9 29/30 Write block 270 len 12288 data: 7 5 5 30/30 Write block 82 len 12288 data: 8 10 8 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.911693284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912453544Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912493543Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912517862Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912545862Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912566131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912590351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91261197Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91263291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912652339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912671738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912738557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912757376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912776306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912792865Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912805405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912817295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912829124Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912840954Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912852574Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912864253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912875913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912887603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912902132Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912914252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912926442Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912938771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912950701Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912962521Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.912975021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91298674Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91300281Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913022889Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913035739Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913047588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913059428Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913071188Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913082878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913094737Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913106427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913120627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913132696Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913144486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913156206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913167935Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913179785Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913191505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913203375Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913215054Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913226774Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913238523Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913250263Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913261973Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913273653Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913285312Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913297032Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913308802Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913322031Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913333901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913345551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91335744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91336927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91338106Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913392779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913404599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913416289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913427998Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913440468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913452648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913464428Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913476347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913488107Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913499937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913511626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913525026Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913537146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913548945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913560705Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913572445Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913584214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913595824Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913607494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913619313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913631003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913643023Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913654792Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913666562Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913678312Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913690002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913701791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913713611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913725331Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91373852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91375058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91376244Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913774169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913785929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913797699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913809579Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913821388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913833188Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913844827Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913856477Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913868247Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913879877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913891696Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913903576Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913915486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913927195Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913940245Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913952505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913964434Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913976214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.913987904Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914004893Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914023013Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914035353Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914047162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914070581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914083461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914095491Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.914107191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:13.91411888Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1488,"session_id":"4fa371ce-b961-48e6-9164-49f3a30802b4"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 146 146 146 F F F 1 020-039 74 74 74 142 142 142 F F F 2 040-059 71 71 71 136 136 136 F F F 3 060-079 73 73 73 140 140 140 F F F 4 080-099 74 74 74 147 147 147 T T T 5 100-119 64 64 64 123 123 123 F F F 6 120-139 74 74 74 145 145 145 T T T 7 140-159 74 74 74 142 142 142 T T T 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 74 74 74 146 146 146 F F F 12 240-259 72 72 72 139 139 139 F F F 13 260-279 73 73 73 140 140 140 T T T 14 280-299 71 71 71 136 136 136 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 74 74 74 142 142 142 F F F 18 360-379 73 73 73 140 140 140 F F F 19 380-399 71 71 71 136 136 136 F F F 20 400-419 74 74 74 145 145 145 F F F 21 420-439 72 72 72 139 139 139 F F F 22 440-459 73 73 73 140 140 140 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 74 74 74 143 143 143 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 72 72 72 139 139 139 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 73 73 73 140 140 140 F F F Max gen: 74, Max flush: 147 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 146 146 146 F F F 1 020-039 74 74 74 142 142 142 F F F 2 040-059 71 71 71 136 136 136 F F F 3 060-079 73 73 73 140 140 140 F F F 4 080-099 75 75 75 148 148 148 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 75 75 75 148 148 148 F F F 7 140-159 75 75 75 148 148 148 F F F 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 74 74 74 146 146 146 F F F 12 240-259 72 72 72 139 139 139 F F F 13 260-279 75 75 75 148 148 148 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 72 72 72 137 137 137 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 74 74 74 142 142 142 F F F 18 360-379 73 73 73 140 140 140 F F F 19 380-399 71 71 71 136 136 136 F F F 20 400-419 74 74 74 145 145 145 F F F 21 420-439 72 72 72 139 139 139 F F F 22 440-459 73 73 73 140 140 140 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 74 74 74 143 143 143 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 72 72 72 139 139 139 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 73 73 73 140 140 140 F F F Max gen: 75, Max flush: 148 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.176526098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.221111984Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.221902203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.221955031Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.221974621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.22198949Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} {"msg":"Crucible 53ead1bb-0cdc-4b3e-bead-976549ce2b8f has session id: 8b559387-938c-4aa5-8a0e-da431be45fc8","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.222618964Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Upstairs opts: Upstairs UUID: 53ead1bb-0cdc-4b3e-bead-976549ce2b8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.222665593Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Crucible stats registered with UUID: 53ead1bb-0cdc-4b3e-bead-976549ce2b8f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.222690322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:18.222712051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223078102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223102621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} {"msg":"53ead1bb-0cdc-4b3e-bead-976549ce2b8f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223177419Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223194479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"0","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223216948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"0","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223230848Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"1","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223245057Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"1","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223257917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"2","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223329895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"2","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223374724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223402073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223425773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:64702)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223449362Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36162)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223635447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40018)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.223677486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.224359038Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"0","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.224393187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"2","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.224838495Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"1","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.225011871Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"0","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.22504153Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"2","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.225511468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"1","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.225678653Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.225705213Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[0]R flush_numbers[0..12]: [146, 142, 136, 140, 148, 123, 148, 148, 132, 134, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226048044Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[0]R generation[0..12]: [74, 74, 71, 73, 75, 64, 75, 75, 68, 69, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226067363Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226081763Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[1]R flush_numbers[0..12]: [146, 142, 136, 140, 148, 123, 148, 148, 132, 134, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226095452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[1]R generation[0..12]: [74, 74, 71, 73, 75, 64, 75, 75, 68, 69, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226108662Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226122192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[2]R flush_numbers[0..12]: [146, 142, 136, 140, 148, 123, 148, 148, 132, 134, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226135401Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[2]R generation[0..12]: [74, 74, 71, 73, 75, 64, 75, 75, 68, 69, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226148491Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226161991Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.22617447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.22618741Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.2261999Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226225579Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"":"downstairs","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226241209Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226252928Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"0","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226265888Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"1","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226278717Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"client":"2","":"downstairs","session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226291517Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226303487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"53ead1bb-0cdc-4b3e-bead-976549ce2b8f is now active with session: 8b559387-938c-4aa5-8a0e-da431be45fc8","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226316176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226327676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496,"session_id":"8b559387-938c-4aa5-8a0e-da431be45fc8"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226339376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} {"msg":"Activated sub_volume 53ead1bb-0cdc-4b3e-bead-976549ce2b8f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:18.226350376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 12288 data: 8 8 5 02/30 Read block 488 len 32768 03/30 Read block 251 len 8192 04/30 Read block 337 len 16384 05/30 Read block 440 len 4096 06/30 Write block 584 len 28672 data: 5 5 4 7 4 3 5 07/30 Read block 235 len 12288 08/30 Read block 518 len 24576 09/30 Read block 562 len 32768 10/30 Write block 95 len 28672 data: 9 11 11 10 10 10 10 11/30 Flush 12/30 Write block 86 len 24576 data: 8 8 9 11 8 8 13/30 Write block 187 len 12288 data: 3 4 5 14/30 Read block 326 len 16384 15/30 Write block 108 len 16384 data: 6 7 7 6 16/30 Write block 157 len 8192 data: 7 6 17/30 Write block 545 len 28672 data: 11 10 14 13 12 12 12 18/30 Read block 33 len 36864 19/30 Read block 447 len 20480 20/30 Read block 226 len 8192 21/30 Write block 64 len 24576 data: 7 7 8 6 6 6 22/30 Read block 12 len 32768 23/30 Read block 547 len 4096 24/30 Read block 571 len 40960 25/30 Read block 218 len 4096 26/30 Write block 304 len 28672 data: 5 6 6 6 6 8 8 27/30 Write block 392 len 4096 data: 8 28/30 Write block 453 len 28672 data: 7 7 7 6 6 7 6 29/30 Read block 87 len 4096 30/30 Write block 486 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:18.246218062Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 146 146 146 F F F 1 020-039 74 74 74 142 142 142 F F F 2 040-059 71 71 71 136 136 136 F F F 3 060-079 73 73 73 140 140 140 T T T 4 080-099 76 76 76 149 149 149 T T T 5 100-119 76 76 76 149 149 149 T T T 6 120-139 75 75 75 148 148 148 F F F 7 140-159 76 76 76 149 149 149 T T T 8 160-179 68 68 68 132 132 132 F F F 9 180-199 69 69 69 134 134 134 T T T 10 200-219 71 71 71 136 136 136 F F F 11 220-239 74 74 74 146 146 146 F F F 12 240-259 72 72 72 139 139 139 F F F 13 260-279 75 75 75 148 148 148 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 72 72 72 137 137 137 T T T 16 320-339 74 74 74 143 143 143 F F F 17 340-359 74 74 74 142 142 142 F F F 18 360-379 73 73 73 140 140 140 F F F 19 380-399 71 71 71 136 136 136 T T T 20 400-419 74 74 74 145 145 145 F F F 21 420-439 72 72 72 139 139 139 F F F 22 440-459 73 73 73 140 140 140 T T T 23 460-479 71 71 71 136 136 136 F F F 24 480-499 74 74 74 143 143 143 T T T 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 72 72 72 139 139 139 T T T 28 560-579 73 73 73 140 140 140 F F F 29 580-599 76 76 76 149 149 149 F F F Max gen: 76, Max flush: 149 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 146 146 146 F F F 1 020-039 74 74 74 142 142 142 F F F 2 040-059 71 71 71 136 136 136 F F F 3 060-079 77 77 77 150 150 150 F F F 4 080-099 77 77 77 150 150 150 F F F 5 100-119 77 77 77 150 150 150 F F F 6 120-139 75 75 75 148 148 148 F F F 7 140-159 77 77 77 150 150 150 F F F 8 160-179 68 68 68 132 132 132 F F F 9 180-199 77 77 77 150 150 150 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 74 74 74 146 146 146 F F F 12 240-259 72 72 72 139 139 139 F F F 13 260-279 75 75 75 148 148 148 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 77 77 77 150 150 150 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 74 74 74 142 142 142 F F F 18 360-379 73 73 73 140 140 140 F F F 19 380-399 77 77 77 150 150 150 F F F 20 400-419 74 74 74 145 145 145 F F F 21 420-439 72 72 72 139 139 139 F F F 22 440-459 77 77 77 150 150 150 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 77 77 77 150 150 150 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 F F F 27 540-559 77 77 77 150 150 150 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 76 76 76 149 149 149 F F F Max gen: 77, Max flush: 150 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.512981516Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.559573778Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.560230231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.56026527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.5602796Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.560300449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} {"msg":"Crucible d47f58e7-bb61-4906-96a8-cde3b3b6070b has session id: e53ef411-c15f-4b5c-ae1f-f6459db49e3c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.560946002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Upstairs opts: Upstairs UUID: d47f58e7-bb61-4906-96a8-cde3b3b6070b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.560987241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Crucible stats registered with UUID: d47f58e7-bb61-4906-96a8-cde3b3b6070b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561003571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:22.56102665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561504698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561537757Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} {"msg":"d47f58e7-bb61-4906-96a8-cde3b3b6070b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561606795Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561632494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"0","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561678053Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"0","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561707142Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"1","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561734092Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"1","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561756581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"2","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.56178273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"2","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561808179Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561832239Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561856418Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39063)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.561882588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56094)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.562049433Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43486)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.562095672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.562668557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"0","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.562704036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"1","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.562869702Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"2","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.56332719Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"0","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.563363709Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"1","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.563412357Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"2","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.563931954Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.563967573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[0]R flush_numbers[0..12]: [146, 142, 136, 150, 150, 150, 148, 150, 132, 150, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.5640699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[0]R generation[0..12]: [74, 74, 71, 77, 77, 77, 75, 77, 68, 77, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564090029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564105049Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[1]R flush_numbers[0..12]: [146, 142, 136, 150, 150, 150, 148, 150, 132, 150, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564118979Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[1]R generation[0..12]: [74, 74, 71, 77, 77, 77, 75, 77, 68, 77, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564132178Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564148828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[2]R flush_numbers[0..12]: [146, 142, 136, 150, 150, 150, 148, 150, 132, 150, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564162708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[2]R generation[0..12]: [74, 74, 71, 77, 77, 77, 75, 77, 68, 77, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564176077Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564190047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564202627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2025-03-19T17:03:22.564215626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564234575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564247045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"":"downstairs","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564259665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564271405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"0","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564284224Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"1","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564297034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"client":"2","":"downstairs","session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564309914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564321753Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"d47f58e7-bb61-4906-96a8-cde3b3b6070b is now active with session: e53ef411-c15f-4b5c-ae1f-f6459db49e3c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564334203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564348063Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504,"session_id":"e53ef411-c15f-4b5c-ae1f-f6459db49e3c"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564360452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} {"msg":"Activated sub_volume d47f58e7-bb61-4906-96a8-cde3b3b6070b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:22.564371352Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 535 len 28672 data: 7 8 8 9 10 8 7 04/30 Read block 97 len 20480 05/30 Write block 438 len 8192 data: 6 5 06/30 Write block 520 len 28672 data: 4 5 5 6 5 5 5 07/30 Read block 421 len 32768 08/30 Write block 176 len 4096 data: 6 09/30 Read block 299 len 28672 10/30 Read block 510 len 4096 11/30 Write block 143 len 20480 data: 8 7 9 9 9 12/30 Read block 423 len 16384 13/30 Read block 423 len 24576 14/30 Read block 409 len 4096 15/30 Read block 402 len 20480 16/30 Read block 416 len 32768 17/30 Write block 15 len 28672 data: 7 6 7 8 8 7 7 18/30 Read block 473 len 12288 19/30 Read block 322 len 36864 20/30 Write block 89 len 12288 data: 12 9 9 21/30 Write block 257 len 32768 data: 4 3 7 7 11 12 9 8 22/30 Write block 415 len 28672 data: 7 9 8 8 10 10 7 23/30 Read block 7 len 4096 24/30 Write block 170 len 8192 data: 7 7 25/30 Write block 398 len 36864 data: 11 9 10 10 8 9 7 6 6 26/30 Write block 363 len 24576 data: 10 7 6 7 8 8 27/30 Read block 356 len 28672 28/30 Read block 321 len 4096 29/30 Write block 341 len 36864 data: 10 11 10 11 11 11 10 11 10 30/30 Write block 435 len 32768 data: 6 6 6 7 6 5 5 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:22.584288517Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 146 146 146 T T T 1 020-039 74 74 74 142 142 142 T T T 2 040-059 71 71 71 136 136 136 F F F 3 060-079 77 77 77 150 150 150 F F F 4 080-099 77 77 77 150 150 150 T T T 5 100-119 77 77 77 150 150 150 F F F 6 120-139 75 75 75 148 148 148 F F F 7 140-159 77 77 77 150 150 150 T T T 8 160-179 68 68 68 132 132 132 T T T 9 180-199 77 77 77 150 150 150 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 74 74 74 146 146 146 F F F 12 240-259 72 72 72 139 139 139 T T T 13 260-279 75 75 75 148 148 148 T T T 14 280-299 71 71 71 136 136 136 F F F 15 300-319 77 77 77 150 150 150 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 74 74 74 142 142 142 T T T 18 360-379 73 73 73 140 140 140 T T T 19 380-399 77 77 77 150 150 150 T T T 20 400-419 74 74 74 145 145 145 T T T 21 420-439 72 72 72 139 139 139 T T T 22 440-459 77 77 77 150 150 150 T T T 23 460-479 71 71 71 136 136 136 F F F 24 480-499 77 77 77 150 150 150 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 69 69 69 134 134 134 T T T 27 540-559 77 77 77 150 150 150 T T T 28 560-579 73 73 73 140 140 140 F F F 29 580-599 76 76 76 149 149 149 F F F Max gen: 77, Max flush: 150 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 151 151 151 F F F 1 020-039 78 78 78 151 151 151 F F F 2 040-059 71 71 71 136 136 136 F F F 3 060-079 77 77 77 150 150 150 F F F 4 080-099 78 78 78 151 151 151 F F F 5 100-119 77 77 77 150 150 150 F F F 6 120-139 75 75 75 148 148 148 F F F 7 140-159 78 78 78 151 151 151 F F F 8 160-179 78 78 78 151 151 151 F F F 9 180-199 77 77 77 150 150 150 F F F 10 200-219 71 71 71 136 136 136 F F F 11 220-239 74 74 74 146 146 146 F F F 12 240-259 78 78 78 151 151 151 F F F 13 260-279 78 78 78 151 151 151 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 77 77 77 150 150 150 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 78 78 78 151 151 151 F F F 18 360-379 78 78 78 151 151 151 F F F 19 380-399 78 78 78 151 151 151 F F F 20 400-419 78 78 78 151 151 151 F F F 21 420-439 78 78 78 151 151 151 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 77 77 77 150 150 150 F F F 25 500-519 71 71 71 136 136 136 F F F 26 520-539 78 78 78 151 151 151 F F F 27 540-559 78 78 78 151 151 151 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 76 76 76 149 149 149 F F F Max gen: 78, Max flush: 151 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.870790384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.915281712Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.916016082Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.916052791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.916072311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.91609241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} {"msg":"Crucible 407eb9f7-a604-4a78-a701-28ad5ac86646 has session id: ad28b231-cad7-406d-a87b-6d759c608694","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.916628326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Upstairs opts: Upstairs UUID: 407eb9f7-a604-4a78-a701-28ad5ac86646, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.916720843Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Crucible stats registered with UUID: 407eb9f7-a604-4a78-a701-28ad5ac86646","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.916751233Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:36.916767002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.91724334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917275459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} {"msg":"407eb9f7-a604-4a78-a701-28ad5ac86646 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917333367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917367997Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"0","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917418995Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"0","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917450894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"1","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917482083Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"1","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917509403Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"2","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917540832Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"2","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917585761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.91761268Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917642009Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"ds_connection connected from Ok(127.0.0.1:59473)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917700918Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"ds_connection connected from Ok(127.0.0.1:36365)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917749056Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"ds_connection connected from Ok(127.0.0.1:43233)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.917785275Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.918478437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"0","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.918507047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"2","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.918551005Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"1","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.919229117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"2","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.919259817Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"0","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.919275226Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"1","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.919852411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.919996937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[0]R flush_numbers[0..12]: [151, 151, 136, 150, 151, 150, 148, 151, 151, 150, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920060815Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[0]R generation[0..12]: [78, 78, 71, 77, 78, 77, 75, 78, 78, 77, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920079875Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920094295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[1]R flush_numbers[0..12]: [151, 151, 136, 150, 151, 150, 148, 151, 151, 150, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920107984Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[1]R generation[0..12]: [78, 78, 71, 77, 78, 77, 75, 78, 78, 77, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920121494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920135304Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[2]R flush_numbers[0..12]: [151, 151, 136, 150, 151, 150, 148, 151, 151, 150, 136, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920148983Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[2]R generation[0..12]: [78, 78, 71, 77, 78, 77, 75, 78, 78, 77, 71, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920162203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920175812Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Max found gen is 79","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920192492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Generation requested: 80 >= found:79","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920205872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-19T17:03:36.920225361Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920238161Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"":"downstairs","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920251061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.92026276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"0","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.92027555Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"1","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920288289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"client":"2","":"downstairs","session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920301609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920313459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"407eb9f7-a604-4a78-a701-28ad5ac86646 is now active with session: ad28b231-cad7-406d-a87b-6d759c608694","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920326059Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920337658Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512,"session_id":"ad28b231-cad7-406d-a87b-6d759c608694"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920353408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} {"msg":"Activated sub_volume 407eb9f7-a604-4a78-a701-28ad5ac86646","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:36.920364478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 32768 data: 7 8 10 9 8 7 7 7 02/30 Read block 502 len 4096 03/30 Read block 449 len 4096 04/30 Read block 293 len 32768 05/30 Read block 56 len 20480 06/30 Read block 545 len 40960 07/30 Write block 211 len 12288 data: 7 8 5 08/30 Read block 261 len 32768 09/30 Read block 217 len 20480 10/30 Read block 461 len 12288 11/30 Read block 266 len 36864 12/30 Read block 460 len 32768 13/30 Read block 322 len 36864 14/30 Write block 524 len 8192 data: 6 6 15/30 Write block 518 len 32768 data: 4 5 5 6 6 7 7 7 16/30 Write block 232 len 28672 data: 3 4 4 7 7 7 6 17/30 Write block 538 len 20480 data: 10 11 9 8 9 18/30 Read block 334 len 40960 19/30 Write block 501 len 20480 data: 2 2 3 4 4 20/30 Read block 72 len 36864 21/30 Write block 139 len 40960 data: 8 8 9 7 9 8 10 10 10 6 22/30 Flush 23/30 Write block 51 len 28672 data: 6 7 9 9 7 6 7 24/30 Write block 21 len 28672 data: 8 8 6 7 8 8 9 25/30 Read block 513 len 28672 26/30 Read block 398 len 12288 27/30 Read block 543 len 16384 28/30 Read block 221 len 32768 29/30 Write block 90 len 16384 data: 10 10 8 11 30/30 Read block 410 len 20480 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:36.942609612Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 151 151 151 F F F 1 020-039 78 78 78 151 151 151 T T T 2 040-059 71 71 71 136 136 136 T T T 3 060-079 77 77 77 150 150 150 F F F 4 080-099 78 78 78 151 151 151 T T T 5 100-119 77 77 77 150 150 150 F F F 6 120-139 80 80 80 152 152 152 F F F 7 140-159 80 80 80 152 152 152 F F F 8 160-179 78 78 78 151 151 151 F F F 9 180-199 77 77 77 150 150 150 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 78 78 78 151 151 151 F F F 13 260-279 78 78 78 151 151 151 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 77 77 77 150 150 150 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 78 78 78 151 151 151 F F F 18 360-379 80 80 80 152 152 152 F F F 19 380-399 80 80 80 152 152 152 F F F 20 400-419 78 78 78 151 151 151 F F F 21 420-439 78 78 78 151 151 151 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 77 77 77 150 150 150 F F F 25 500-519 80 80 80 152 152 152 F F F 26 520-539 80 80 80 152 152 152 F F F 27 540-559 80 80 80 152 152 152 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 76 76 76 149 149 149 F F F Max gen: 80, Max flush: 152 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 151 151 151 F F F 1 020-039 81 81 81 153 153 153 F F F 2 040-059 81 81 81 153 153 153 F F F 3 060-079 77 77 77 150 150 150 F F F 4 080-099 81 81 81 153 153 153 F F F 5 100-119 77 77 77 150 150 150 F F F 6 120-139 80 80 80 152 152 152 F F F 7 140-159 80 80 80 152 152 152 F F F 8 160-179 78 78 78 151 151 151 F F F 9 180-199 77 77 77 150 150 150 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 78 78 78 151 151 151 F F F 13 260-279 78 78 78 151 151 151 F F F 14 280-299 71 71 71 136 136 136 F F F 15 300-319 77 77 77 150 150 150 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 78 78 78 151 151 151 F F F 18 360-379 80 80 80 152 152 152 F F F 19 380-399 80 80 80 152 152 152 F F F 20 400-419 78 78 78 151 151 151 F F F 21 420-439 78 78 78 151 151 151 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 71 71 71 136 136 136 F F F 24 480-499 77 77 77 150 150 150 F F F 25 500-519 80 80 80 152 152 152 F F F 26 520-539 80 80 80 152 152 152 F F F 27 540-559 80 80 80 152 152 152 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 76 76 76 149 149 149 F F F Max gen: 81, Max flush: 153 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.175128327Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:41.217869561Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.21830147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.24942345Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.250070673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.250105672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.250119242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.250133211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} {"msg":"Crucible f525506c-9204-47cc-9c3f-e8a0b9d8d469 has session id: ff0c785d-83d5-40c8-84b8-6bd8dfb6847c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.250776295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Upstairs opts: Upstairs UUID: f525506c-9204-47cc-9c3f-e8a0b9d8d469, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.250822583Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Crucible stats registered with UUID: f525506c-9204-47cc-9c3f-e8a0b9d8d469","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.250846773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:41.250867632Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.25132007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251346849Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} {"msg":"f525506c-9204-47cc-9c3f-e8a0b9d8d469 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251401668Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251426277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"0","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251449557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"0","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251463916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"1","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251478126Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"1","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251491265Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"2","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251504855Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"2","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251517685Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251531175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251547874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52569)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251611382Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57610)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251829087Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35460)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.251868536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.252381952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"2","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.252417851Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"1","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.252574897Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"0","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253272869Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"1","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253301538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"2","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253317588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"0","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253865733Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253891012Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[0]R flush_numbers[0..12]: [151, 153, 153, 150, 153, 150, 152, 152, 151, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253919212Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[0]R generation[0..12]: [78, 81, 81, 77, 81, 77, 80, 80, 78, 77, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253934231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.253948331Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[1]R flush_numbers[0..12]: [151, 153, 153, 150, 153, 150, 152, 152, 151, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.25396542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[1]R generation[0..12]: [78, 81, 81, 77, 81, 77, 80, 80, 78, 77, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.25397921Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.25399262Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[2]R flush_numbers[0..12]: [151, 153, 153, 150, 153, 150, 152, 152, 151, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254006089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[2]R generation[0..12]: [78, 81, 81, 77, 81, 77, 80, 80, 78, 77, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254019219Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254032859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254045488Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254058238Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254070778Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254083187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"":"downstairs","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254095997Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254107597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"0","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:03:41.254120876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"1","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254138856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"client":"2","":"downstairs","session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254153426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254165435Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"f525506c-9204-47cc-9c3f-e8a0b9d8d469 is now active with session: ff0c785d-83d5-40c8-84b8-6bd8dfb6847c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254177785Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254189594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520,"session_id":"ff0c785d-83d5-40c8-84b8-6bd8dfb6847c"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254201444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} {"msg":"Activated sub_volume f525506c-9204-47cc-9c3f-e8a0b9d8d469","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:41.254212344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 55 len 4096 data: 8 02/30 Read block 236 len 8192 03/30 Write block 478 len 40960 data: 6 7 9 8 6 6 6 6 8 6 04/30 Write block 542 len 28672 data: 10 12 11 12 11 15 14 05/30 Read block 574 len 36864 06/30 Read block 126 len 36864 07/30 Read block 489 len 20480 08/30 Read block 235 len 24576 09/30 Read block 465 len 24576 10/30 Flush 11/30 Write block 594 len 12288 data: 5 5 5 12/30 Flush 13/30 Write block 289 len 12288 data: 3 3 3 14/30 Write block 5 len 12288 data: 4 5 5 15/30 Write block 493 len 36864 data: 6 8 9 9 7 6 3 2 3 16/30 Read block 319 len 8192 17/30 Read block 327 len 20480 18/30 Read block 128 len 32768 19/30 Write block 503 len 36864 data: 4 5 5 4 5 9 9 9 12 20/30 Write block 58 len 28672 data: 5 6 6 6 8 8 8 21/30 Write block 374 len 20480 data: 10 6 6 8 9 22/30 Read block 188 len 8192 23/30 Read block 352 len 24576 24/30 Read block 412 len 36864 25/30 Read block 358 len 40960 26/30 Write block 248 len 36864 data: 11 10 9 7 7 5 5 4 3 27/30 Write block 393 len 20480 data: 8 11 11 9 10 28/30 Read block 435 len 4096 29/30 Read block 428 len 28672 30/30 Write block 344 len 12288 data: 12 12 12 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:41.274910489Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 151 151 151 T T T 1 020-039 81 81 81 153 153 153 F F F 2 040-059 82 82 82 154 154 154 T T T 3 060-079 77 77 77 150 150 150 T T T 4 080-099 81 81 81 153 153 153 F F F 5 100-119 77 77 77 150 150 150 F F F 6 120-139 80 80 80 152 152 152 F F F 7 140-159 80 80 80 152 152 152 F F F 8 160-179 78 78 78 151 151 151 F F F 9 180-199 77 77 77 150 150 150 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 78 78 78 151 151 151 T T T 13 260-279 78 78 78 151 151 151 F F F 14 280-299 71 71 71 136 136 136 T T T 15 300-319 77 77 77 150 150 150 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 78 78 78 151 151 151 T T T 18 360-379 80 80 80 152 152 152 T T T 19 380-399 80 80 80 152 152 152 T T T 20 400-419 78 78 78 151 151 151 F F F 21 420-439 78 78 78 151 151 151 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 82 82 82 154 154 154 F F F 24 480-499 82 82 82 154 154 154 T T T 25 500-519 80 80 80 152 152 152 T T T 26 520-539 80 80 80 152 152 152 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 82 82 82 155 155 155 F F F Max gen: 82, Max flush: 155 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 156 156 156 F F F 1 020-039 81 81 81 153 153 153 F F F 2 040-059 83 83 83 156 156 156 F F F 3 060-079 83 83 83 156 156 156 F F F 4 080-099 81 81 81 153 153 153 F F F 5 100-119 77 77 77 150 150 150 F F F 6 120-139 80 80 80 152 152 152 F F F 7 140-159 80 80 80 152 152 152 F F F 8 160-179 78 78 78 151 151 151 F F F 9 180-199 77 77 77 150 150 150 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 83 83 83 156 156 156 F F F 13 260-279 78 78 78 151 151 151 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 77 77 77 150 150 150 F F F 16 320-339 74 74 74 143 143 143 F F F 17 340-359 83 83 83 156 156 156 F F F 18 360-379 83 83 83 156 156 156 F F F 19 380-399 83 83 83 156 156 156 F F F 20 400-419 78 78 78 151 151 151 F F F 21 420-439 78 78 78 151 151 151 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 82 82 82 154 154 154 F F F 24 480-499 83 83 83 156 156 156 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 80 80 80 152 152 152 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 73 73 73 140 140 140 F F F 29 580-599 82 82 82 155 155 155 F F F Max gen: 83, Max flush: 156 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.570552841Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.614473974Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.615192815Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.615228474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.615245204Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.615267503Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} {"msg":"Crucible 11519beb-5b35-44f1-9041-0211785e96a3 has session id: 7932cf80-0575-4caa-8f45-575b1959f3c2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.615716061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Upstairs opts: Upstairs UUID: 11519beb-5b35-44f1-9041-0211785e96a3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.61576203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Crucible stats registered with UUID: 11519beb-5b35-44f1-9041-0211785e96a3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.615787729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:45.615806949Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616357405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616382284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} {"msg":"11519beb-5b35-44f1-9041-0211785e96a3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616446782Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616467682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"0","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616490421Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"0","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616503961Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"1","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.61651999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"1","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.61654228Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"2","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616567239Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"2","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616591518Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616639567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616669196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51858)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616699196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:32857)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616727715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36935)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.616773603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.617433986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"0","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.617462605Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"1","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.618125798Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"1","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.618171087Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"0","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.618713932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.61880851Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.619088553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"2","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.619591289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"2","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[0]R flush_numbers[0..12]: [156, 153, 156, 156, 153, 150, 152, 152, 151, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620083576Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[0]R generation[0..12]: [83, 81, 83, 83, 81, 77, 80, 80, 78, 77, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620106886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620121855Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[1]R flush_numbers[0..12]: [156, 153, 156, 156, 153, 150, 152, 152, 151, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620135945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[1]R generation[0..12]: [83, 81, 83, 83, 81, 77, 80, 80, 78, 77, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620149494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620163394Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[2]R flush_numbers[0..12]: [156, 153, 156, 156, 153, 150, 152, 152, 151, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620176864Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[2]R generation[0..12]: [83, 81, 83, 83, 81, 77, 80, 80, 78, 77, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620190013Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620203673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620219343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620232462Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620244992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620257602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"":"downstairs","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620270461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:03:45.620282041Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"0","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.62030395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"1","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.62031685Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"client":"2","":"downstairs","session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.62032987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.62034174Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"11519beb-5b35-44f1-9041-0211785e96a3 is now active with session: 7932cf80-0575-4caa-8f45-575b1959f3c2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620354219Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620365959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528,"session_id":"7932cf80-0575-4caa-8f45-575b1959f3c2"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620377948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} {"msg":"Activated sub_volume 11519beb-5b35-44f1-9041-0211785e96a3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:45.620388828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 170 len 28672 data: 8 8 7 8 7 8 7 02/30 Write block 243 len 40960 data: 9 5 7 8 12 12 11 10 8 8 03/30 Write block 562 len 12288 data: 5 7 9 04/30 Flush 05/30 Write block 422 len 40960 data: 7 8 8 7 5 4 6 5 6 7 06/30 Flush 07/30 Write block 567 len 32768 data: 8 10 8 7 4 5 5 4 08/30 Write block 475 len 24576 data: 7 6 6 7 8 10 09/30 Read block 432 len 4096 10/30 Read block 83 len 28672 11/30 Read block 25 len 40960 12/30 Read block 134 len 4096 13/30 Flush 14/30 Write block 143 len 4096 data: 10 15/30 Read block 215 len 36864 16/30 Read block 20 len 20480 17/30 Read block 168 len 20480 18/30 Read block 318 len 24576 19/30 Write block 178 len 20480 data: 5 6 5 4 3 20/30 Read block 175 len 12288 21/30 Read block 244 len 4096 22/30 Flush 23/30 Write block 329 len 8192 data: 6 5 24/30 Write block 65 len 20480 data: 8 9 7 7 7 25/30 Write block 375 len 32768 data: 7 7 9 10 11 10 9 8 26/30 Write block 552 len 12288 data: 10 11 9 27/30 Write block 304 len 40960 data: 6 7 7 7 7 9 9 8 8 8 28/30 Write block 104 len 24576 data: 10 11 8 7 7 8 29/30 Read block 74 len 40960 30/30 Write block 417 len 40960 data: 9 9 11 11 8 8 9 9 8 6 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:45.640960406Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 156 156 156 F F F 1 020-039 81 81 81 153 153 153 F F F 2 040-059 83 83 83 156 156 156 F F F 3 060-079 83 83 83 156 156 156 T T T 4 080-099 81 81 81 153 153 153 F F F 5 100-119 77 77 77 150 150 150 T T T 6 120-139 80 80 80 152 152 152 F F F 7 140-159 84 84 84 160 160 160 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 84 84 84 160 160 160 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 84 84 84 157 157 157 F F F 13 260-279 78 78 78 151 151 151 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 77 77 77 150 150 150 T T T 16 320-339 74 74 74 143 143 143 T T T 17 340-359 83 83 83 156 156 156 F F F 18 360-379 83 83 83 156 156 156 T T T 19 380-399 83 83 83 156 156 156 T T T 20 400-419 78 78 78 151 151 151 T T T 21 420-439 84 84 84 158 158 158 T T T 22 440-459 78 78 78 151 151 151 F F F 23 460-479 84 84 84 159 159 159 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 80 80 80 152 152 152 F F F 27 540-559 82 82 82 154 154 154 T T T 28 560-579 84 84 84 159 159 159 F F F 29 580-599 82 82 82 155 155 155 F F F Max gen: 84, Max flush: 160 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 156 156 156 F F F 1 020-039 81 81 81 153 153 153 F F F 2 040-059 83 83 83 156 156 156 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 81 81 81 153 153 153 F F F 5 100-119 85 85 85 161 161 161 F F F 6 120-139 80 80 80 152 152 152 F F F 7 140-159 84 84 84 160 160 160 F F F 8 160-179 84 84 84 160 160 160 F F F 9 180-199 84 84 84 160 160 160 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 84 84 84 157 157 157 F F F 13 260-279 78 78 78 151 151 151 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 85 85 85 161 161 161 F F F 16 320-339 85 85 85 161 161 161 F F F 17 340-359 83 83 83 156 156 156 F F F 18 360-379 85 85 85 161 161 161 F F F 19 380-399 85 85 85 161 161 161 F F F 20 400-419 85 85 85 161 161 161 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 84 84 84 159 159 159 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 80 80 80 152 152 152 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 84 84 84 159 159 159 F F F 29 580-599 82 82 82 155 155 155 F F F Max gen: 85, Max flush: 161 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.903633418Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:59.945770948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.946194927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.978227553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.978894316Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.978928535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.978941674Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.978955734Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} {"msg":"Crucible deeabca8-c7a3-4097-9aa3-d95fe1eea99d has session id: a112f9f2-d744-43b0-81e9-7ec182a87f81","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.979596047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Upstairs opts: Upstairs UUID: deeabca8-c7a3-4097-9aa3-d95fe1eea99d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.979645376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Crucible stats registered with UUID: deeabca8-c7a3-4097-9aa3-d95fe1eea99d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.979671795Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:03:59.979692855Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980084424Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980107934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} {"msg":"deeabca8-c7a3-4097-9aa3-d95fe1eea99d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980179892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980196821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"0","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980219601Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"0","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980233131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"1","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.98024701Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"1","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.98026025Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"2","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980273959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"2","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980286679Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980303878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980317698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:53034)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980391106Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:51610)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980427215Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"ds_connection connected from Ok(127.0.0.1:44362)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.980456394Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.981186185Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"2","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.981214945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"1","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.981452258Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"0","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.981886377Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"1","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.981916516Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"2","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982189899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"0","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982457622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982544419Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[0]R flush_numbers[0..12]: [156, 153, 156, 161, 153, 161, 152, 160, 160, 160, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982721785Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[0]R generation[0..12]: [83, 81, 83, 85, 81, 85, 80, 84, 84, 84, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982744624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982759564Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[1]R flush_numbers[0..12]: [156, 153, 156, 161, 153, 161, 152, 160, 160, 160, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982773104Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[1]R generation[0..12]: [83, 81, 83, 85, 81, 85, 80, 84, 84, 84, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982786313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982799853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[2]R flush_numbers[0..12]: [156, 153, 156, 161, 153, 161, 152, 160, 160, 160, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982813383Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[2]R generation[0..12]: [83, 81, 83, 85, 81, 85, 80, 84, 84, 84, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982826592Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982840142Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982852771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982865631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982878261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982890841Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"":"downstairs","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982904Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.98291598Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"0","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982928929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"1","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:03:59.982943589Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"client":"2","":"downstairs","session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982965759Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982977878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"deeabca8-c7a3-4097-9aa3-d95fe1eea99d is now active with session: a112f9f2-d744-43b0-81e9-7ec182a87f81","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.982990398Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.983002198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536,"session_id":"a112f9f2-d744-43b0-81e9-7ec182a87f81"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.983014317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} {"msg":"Activated sub_volume deeabca8-c7a3-4097-9aa3-d95fe1eea99d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:03:59.983025577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 394 len 28672 data: 12 12 10 11 12 10 11 02/30 Flush 03/30 Write block 104 len 16384 data: 11 12 9 8 04/30 Read block 458 len 16384 05/30 Write block 177 len 40960 data: 4 6 7 6 5 4 5 5 4 3 06/30 Flush 07/30 Write block 584 len 28672 data: 6 6 5 8 5 4 6 08/30 Read block 429 len 32768 09/30 Read block 444 len 36864 10/30 Read block 361 len 4096 11/30 Read block 18 len 12288 12/30 Read block 590 len 12288 13/30 Write block 565 len 12288 data: 9 9 9 14/30 Read block 65 len 20480 15/30 Read block 440 len 20480 16/30 Flush 17/30 Write block 377 len 20480 data: 10 11 12 11 10 18/30 Write block 18 len 12288 data: 9 9 8 19/30 Read block 293 len 16384 20/30 Read block 150 len 8192 21/30 Read block 295 len 12288 22/30 Write block 340 len 28672 data: 10 11 12 11 13 13 13 23/30 Write block 6 len 28672 data: 6 6 4 5 6 5 5 24/30 Write block 324 len 12288 data: 7 5 7 25/30 Read block 413 len 36864 26/30 Read block 115 len 16384 27/30 Write block 263 len 32768 data: 10 9 8 8 9 9 7 8 28/30 Read block 196 len 28672 29/30 Read block 113 len 40960 30/30 Read block 39 len 12288 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 13 13 14 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:00.005382668Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 156 156 156 T T T 1 020-039 81 81 81 153 153 153 T T T 2 040-059 83 83 83 156 156 156 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 81 81 81 153 153 153 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 80 80 80 152 152 152 F F F 7 140-159 84 84 84 160 160 160 F F F 8 160-179 86 86 86 163 163 163 F F F 9 180-199 86 86 86 163 163 163 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 84 84 84 157 157 157 F F F 13 260-279 78 78 78 151 151 151 T T T 14 280-299 83 83 83 156 156 156 F F F 15 300-319 85 85 85 161 161 161 F F F 16 320-339 85 85 85 161 161 161 T T T 17 340-359 83 83 83 156 156 156 T T T 18 360-379 85 85 85 161 161 161 T T T 19 380-399 86 86 86 162 162 162 T T T 20 400-419 86 86 86 162 162 162 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 84 84 84 159 159 159 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 80 80 80 152 152 152 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 86 86 86 164 164 164 F F F 29 580-599 86 86 86 164 164 164 F F F Max gen: 86, Max flush: 164 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 87 87 87 165 165 165 F F F 2 040-059 83 83 83 156 156 156 F F F 3 060-079 85 85 85 161 161 161 F F F 4 080-099 81 81 81 153 153 153 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 80 80 80 152 152 152 F F F 7 140-159 84 84 84 160 160 160 F F F 8 160-179 86 86 86 163 163 163 F F F 9 180-199 86 86 86 163 163 163 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 80 80 80 152 152 152 F F F 12 240-259 84 84 84 157 157 157 F F F 13 260-279 87 87 87 165 165 165 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 85 85 85 161 161 161 F F F 16 320-339 87 87 87 165 165 165 F F F 17 340-359 87 87 87 165 165 165 F F F 18 360-379 87 87 87 165 165 165 F F F 19 380-399 87 87 87 165 165 165 F F F 20 400-419 86 86 86 162 162 162 F F F 21 420-439 85 85 85 161 161 161 F F F 22 440-459 78 78 78 151 151 151 F F F 23 460-479 84 84 84 159 159 159 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 80 80 80 152 152 152 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 86 86 86 164 164 164 F F F 29 580-599 86 86 86 164 164 164 F F F Max gen: 87, Max flush: 165 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.2896447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.333634511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.334353823Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.334387692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.334401771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.334415711Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} {"msg":"Crucible 3a5a2984-ad38-4629-93d0-b95a97740b82 has session id: da8df856-9682-43e5-8ddb-6aac9f5aaef2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335064694Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Upstairs opts: Upstairs UUID: 3a5a2984-ad38-4629-93d0-b95a97740b82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335108453Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Crucible stats registered with UUID: 3a5a2984-ad38-4629-93d0-b95a97740b82","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335131962Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:04.335151902Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335627349Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335654008Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} {"msg":"3a5a2984-ad38-4629-93d0-b95a97740b82 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335694227Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335727956Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"0","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335756026Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"0","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335770165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"1","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335785035Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"1","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335798834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335825894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335884792Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.335924291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.33595594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:04.3359863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:04.336017469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:04.336046678Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58872)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.336071767Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:44207)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.336250103Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.336285832Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.33674236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"0","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.336770789Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"1","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.337313985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"1","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.337440211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"0","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.337802052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:04.338019776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.337709788Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49727)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.337786936Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.338857198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.339618628Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[0]R flush_numbers[0..12]: [165, 165, 156, 161, 153, 163, 152, 160, 163, 163, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340227892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[0]R generation[0..12]: [87, 87, 83, 85, 81, 86, 80, 84, 86, 86, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340255921Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340272241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[1]R flush_numbers[0..12]: [165, 165, 156, 161, 153, 163, 152, 160, 163, 163, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.34029153Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[1]R generation[0..12]: [87, 87, 83, 85, 81, 86, 80, 84, 86, 86, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.34030567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340319839Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[2]R flush_numbers[0..12]: [165, 165, 156, 161, 153, 163, 152, 160, 163, 163, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340333799Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[2]R generation[0..12]: [87, 87, 83, 85, 81, 86, 80, 84, 86, 86, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340347248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340360948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340373598Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340386347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340398877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340411427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"":"downstairs","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340424406Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340436446Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"0","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340449246Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"1","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340462176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"client":"2","":"downstairs","session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340475125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340487045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"3a5a2984-ad38-4629-93d0-b95a97740b82 is now active with session: da8df856-9682-43e5-8ddb-6aac9f5aaef2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340501644Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340513564Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544,"session_id":"da8df856-9682-43e5-8ddb-6aac9f5aaef2"} Wait for a query_work_queue command to finish before sending IO {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340529924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} {"msg":"Activated sub_volume 3a5a2984-ad38-4629-93d0-b95a97740b82","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:14.340541163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 358 len 20480 data: 7 10 10 10 12 02/30 Read block 411 len 12288 03/30 Write block 431 len 4096 data: 8 04/30 Write block 445 len 20480 data: 6 7 6 5 3 05/30 Read block 207 len 40960 06/30 Write block 405 len 36864 data: 7 7 7 7 5 7 7 7 8 07/30 Write block 301 len 4096 data: 3 08/30 Read block 254 len 16384 09/30 Write block 269 len 8192 data: 8 9 10/30 Write block 529 len 36864 data: 6 5 6 8 7 7 8 9 9 11/30 Write block 268 len 40960 data: 10 9 10 6 6 9 10 10 8 8 12/30 Flush 13/30 Write block 235 len 16384 data: 8 8 8 7 14/30 Flush 15/30 Flush 16/30 Write block 429 len 24576 data: 6 7 9 5 4 5 17/30 Write block 184 len 24576 data: 6 5 4 4 5 6 18/30 Write block 344 len 4096 data: 14 19/30 Read block 566 len 40960 20/30 Write block 127 len 36864 data: 5 5 5 3 3 6 7 7 7 21/30 Read block 335 len 8192 22/30 Read block 501 len 8192 23/30 Write block 251 len 40960 data: 9 9 6 6 5 4 5 4 8 8 24/30 Read block 559 len 32768 25/30 Write block 73 len 36864 data: 3 6 8 7 7 7 7 7 7 26/30 Read block 148 len 28672 27/30 Read block 191 len 24576 28/30 Write block 42 len 32768 data: 7 7 7 7 4 5 5 5 29/30 Write block 444 len 16384 data: 5 7 8 7 30/30 Read block 521 len 8192 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 15 14 13 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:14.361872841Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 87 87 87 165 165 165 F F F 2 040-059 83 83 83 156 156 156 T T T 3 060-079 85 85 85 161 161 161 T T T 4 080-099 81 81 81 153 153 153 T T T 5 100-119 86 86 86 163 163 163 F F F 6 120-139 80 80 80 152 152 152 T T T 7 140-159 84 84 84 160 160 160 F F F 8 160-179 86 86 86 163 163 163 F F F 9 180-199 86 86 86 163 163 163 T T T 10 200-219 80 80 80 152 152 152 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 84 84 84 157 157 157 T T T 13 260-279 88 88 88 166 166 166 T T T 14 280-299 83 83 83 156 156 156 F F F 15 300-319 88 88 88 166 166 166 F F F 16 320-339 87 87 87 165 165 165 F F F 17 340-359 88 88 88 166 166 166 T T T 18 360-379 88 88 88 166 166 166 F F F 19 380-399 87 87 87 165 165 165 F F F 20 400-419 88 88 88 166 166 166 F F F 21 420-439 88 88 88 166 166 166 T T T 22 440-459 88 88 88 166 166 166 T T T 23 460-479 84 84 84 159 159 159 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 86 86 86 164 164 164 F F F 29 580-599 86 86 86 164 164 164 F F F Max gen: 88, Max flush: 167 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 87 87 87 165 165 165 F F F 2 040-059 89 89 89 168 168 168 F F F 3 060-079 89 89 89 168 168 168 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 89 89 89 168 168 168 F F F 7 140-159 84 84 84 160 160 160 F F F 8 160-179 86 86 86 163 163 163 F F F 9 180-199 89 89 89 168 168 168 F F F 10 200-219 80 80 80 152 152 152 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 89 89 89 168 168 168 F F F 13 260-279 89 89 89 168 168 168 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 88 88 88 166 166 166 F F F 16 320-339 87 87 87 165 165 165 F F F 17 340-359 89 89 89 168 168 168 F F F 18 360-379 88 88 88 166 166 166 F F F 19 380-399 87 87 87 165 165 165 F F F 20 400-419 88 88 88 166 166 166 F F F 21 420-439 89 89 89 168 168 168 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 84 84 84 159 159 159 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 88 88 88 166 166 166 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 86 86 86 164 164 164 F F F 29 580-599 86 86 86 164 164 164 F F F Max gen: 89, Max flush: 168 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.626305109Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.669255238Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.669893711Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.66992916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.66994311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.66995757Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} {"msg":"Crucible 9e16da53-f405-4d4e-8d2d-734e55cfe6e8 has session id: 985d7814-d553-4578-9d6b-6c78b8d0ce80","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.670502365Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Upstairs opts: Upstairs UUID: 9e16da53-f405-4d4e-8d2d-734e55cfe6e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.670548954Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Crucible stats registered with UUID: 9e16da53-f405-4d4e-8d2d-734e55cfe6e8","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.670567703Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:28.670630132Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671151278Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671184987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} {"msg":"9e16da53-f405-4d4e-8d2d-734e55cfe6e8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671239316Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671267665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"0","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671307274Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"0","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671340753Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"1","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671372602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"1","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671399692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"2","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671429791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"2","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.67145748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671486109Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671513169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"ds_connection connected from Ok(127.0.0.1:38381)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671539808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"ds_connection connected from Ok(127.0.0.1:47266)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671564637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"ds_connection connected from Ok(127.0.0.1:49641)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.671589956Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.6722119Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"0","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.672245549Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"1","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.672921071Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"1","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.67296089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"0","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.673528995Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.673565754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.673630983Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"2","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674203687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"2","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[0]R flush_numbers[0..12]: [165, 165, 168, 168, 168, 163, 168, 160, 163, 168, 152, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674723864Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[0]R generation[0..12]: [87, 87, 89, 89, 89, 86, 89, 84, 86, 89, 80, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674745183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674760173Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[1]R flush_numbers[0..12]: [165, 165, 168, 168, 168, 163, 168, 160, 163, 168, 152, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674774163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[1]R generation[0..12]: [87, 87, 89, 89, 89, 86, 89, 84, 86, 89, 80, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674788412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674801982Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[2]R flush_numbers[0..12]: [165, 165, 168, 168, 168, 163, 168, 160, 163, 168, 152, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674815222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[2]R generation[0..12]: [87, 87, 89, 89, 89, 86, 89, 84, 86, 89, 80, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674828511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674842071Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674854501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.67487059Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.67488356Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674902939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"":"downstairs","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674915689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674927408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"0","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674940258Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"1","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674952908Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"client":"2","":"downstairs","session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674965838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674977907Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"9e16da53-f405-4d4e-8d2d-734e55cfe6e8 is now active with session: 985d7814-d553-4578-9d6b-6c78b8d0ce80","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.674990277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.675001906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552,"session_id":"985d7814-d553-4578-9d6b-6c78b8d0ce80"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.675013786Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} {"msg":"Activated sub_volume 9e16da53-f405-4d4e-8d2d-734e55cfe6e8","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:28.675024706Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 43 len 8192 data: 8 8 02/30 Write block 177 len 32768 data: 5 7 8 7 6 5 6 7 03/30 Read block 349 len 16384 04/30 Write block 166 len 24576 data: 7 6 6 6 9 9 05/30 Read block 225 len 24576 06/30 Write block 533 len 20480 data: 8 8 9 10 10 07/30 Read block 84 len 8192 08/30 Flush 09/30 Write block 340 len 4096 data: 11 10/30 Read block 187 len 8192 11/30 Read block 67 len 36864 12/30 Write block 67 len 36864 data: 8 8 8 5 5 3 4 7 9 13/30 Read block 422 len 24576 14/30 Write block 407 len 20480 data: 8 8 6 8 8 15/30 Read block 161 len 32768 16/30 Write block 269 len 20480 data: 10 11 7 7 10 17/30 Write block 463 len 20480 data: 7 4 3 4 4 18/30 Read block 306 len 40960 19/30 Flush 20/30 Write block 581 len 28672 data: 6 6 6 7 7 6 9 21/30 Flush 22/30 Write block 205 len 8192 data: 6 6 23/30 Write block 171 len 32768 data: 10 8 9 8 9 8 6 8 24/30 Read block 1 len 12288 25/30 Read block 409 len 28672 26/30 Read block 450 len 32768 27/30 Read block 361 len 28672 28/30 Read block 262 len 32768 29/30 Read block 389 len 32768 30/30 Write block 464 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:28.696045062Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 87 87 87 165 165 165 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 90 90 90 170 170 170 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 89 89 89 168 168 168 F F F 7 140-159 84 84 84 160 160 160 F F F 8 160-179 90 90 90 169 169 169 T T T 9 180-199 90 90 90 169 169 169 F F F 10 200-219 80 80 80 152 152 152 T T T 11 220-239 88 88 88 167 167 167 F F F 12 240-259 89 89 89 168 168 168 F F F 13 260-279 90 90 90 170 170 170 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 88 88 88 166 166 166 F F F 16 320-339 87 87 87 165 165 165 F F F 17 340-359 90 90 90 170 170 170 F F F 18 360-379 88 88 88 166 166 166 F F F 19 380-399 87 87 87 165 165 165 F F F 20 400-419 90 90 90 170 170 170 F F F 21 420-439 89 89 89 168 168 168 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 90 90 90 170 170 170 T T T 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 86 86 86 164 164 164 F F F 29 580-599 90 90 90 171 171 171 F F F Max gen: 90, Max flush: 171 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 87 87 87 165 165 165 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 90 90 90 170 170 170 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 89 89 89 168 168 168 F F F 7 140-159 84 84 84 160 160 160 F F F 8 160-179 91 91 91 172 172 172 F F F 9 180-199 90 90 90 169 169 169 F F F 10 200-219 91 91 91 172 172 172 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 89 89 89 168 168 168 F F F 13 260-279 90 90 90 170 170 170 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 88 88 88 166 166 166 F F F 16 320-339 87 87 87 165 165 165 F F F 17 340-359 90 90 90 170 170 170 F F F 18 360-379 88 88 88 166 166 166 F F F 19 380-399 87 87 87 165 165 165 F F F 20 400-419 90 90 90 170 170 170 F F F 21 420-439 89 89 89 168 168 168 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 91 91 91 172 172 172 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 83 83 83 156 156 156 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 86 86 86 164 164 164 F F F 29 580-599 90 90 90 171 171 171 F F F Max gen: 91, Max flush: 172 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.919629173Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:32.962354207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.962750707Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.993363471Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.993956755Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.993990874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.994004263Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.994018483Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} {"msg":"Crucible 7ed1251d-4ef9-4684-99b1-ff5bf7941c26 has session id: a3b2db64-c193-450c-aa13-f02e4307af3e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.994578479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Upstairs opts: Upstairs UUID: 7ed1251d-4ef9-4684-99b1-ff5bf7941c26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.994614127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Crucible stats registered with UUID: 7ed1251d-4ef9-4684-99b1-ff5bf7941c26","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.994629227Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:32.994641387Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995069515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995092055Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} {"msg":"7ed1251d-4ef9-4684-99b1-ff5bf7941c26 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995151973Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995169093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"0","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995191062Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"0","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995204452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"1","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995218312Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"1","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995231101Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"2","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995244831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"2","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995257601Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.99527079Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.99529868Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54198)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995349738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43581)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995379007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55130)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.995404887Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.996119638Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"1","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.996150287Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"2","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.996380251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"0","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.996934057Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"1","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.996964796Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"2","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.996991135Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"0","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997598529Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997632608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[0]R flush_numbers[0..12]: [165, 165, 169, 170, 168, 163, 168, 160, 172, 169, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997655677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[0]R generation[0..12]: [87, 87, 90, 90, 89, 86, 89, 84, 91, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997671287Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997685637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[1]R flush_numbers[0..12]: [165, 165, 169, 170, 168, 163, 168, 160, 172, 169, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997702516Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[1]R generation[0..12]: [87, 87, 90, 90, 89, 86, 89, 84, 91, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997717236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997730756Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[2]R flush_numbers[0..12]: [165, 165, 169, 170, 168, 163, 168, 160, 172, 169, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997744345Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[2]R generation[0..12]: [87, 87, 90, 90, 89, 86, 89, 84, 91, 90, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997757545Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997772674Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997795144Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997825343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997846402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997859872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"":"downstairs","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997872632Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997884261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"0","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997897171Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"1","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997909931Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"client":"2","":"downstairs","session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.99792492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.99793707Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"7ed1251d-4ef9-4684-99b1-ff5bf7941c26 is now active with session: a3b2db64-c193-450c-aa13-f02e4307af3e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.99794958Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997961189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560,"session_id":"a3b2db64-c193-450c-aa13-f02e4307af3e"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997973069Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} {"msg":"Activated sub_volume 7ed1251d-4ef9-4684-99b1-ff5bf7941c26","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:32.997983859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 371 len 40960 data: 7 8 9 11 8 8 11 12 13 12 03/30 Write block 505 len 16384 data: 6 5 6 10 04/30 Flush 05/30 Write block 182 len 8192 data: 6 7 06/30 Write block 479 len 4096 data: 9 07/30 Read block 536 len 20480 08/30 Read block 163 len 12288 09/30 Write block 296 len 4096 data: 5 10/30 Write block 247 len 20480 data: 13 13 12 11 10 11/30 Write block 178 len 40960 data: 9 9 8 7 7 8 8 6 5 5 12/30 Flush 13/30 Write block 290 len 40960 data: 4 4 3 6 7 6 6 4 4 4 14/30 Write block 35 len 12288 data: 8 8 7 15/30 Flush 16/30 Write block 69 len 16384 data: 9 6 6 4 17/30 Read block 539 len 12288 18/30 Read block 307 len 12288 19/30 Read block 5 len 16384 20/30 Write block 100 len 36864 data: 11 11 10 12 12 13 10 9 8 21/30 Read block 559 len 36864 22/30 Write block 404 len 20480 data: 8 8 8 9 9 23/30 Write block 570 len 40960 data: 8 5 6 6 5 6 6 7 8 7 24/30 Write block 474 len 8192 data: 7 8 25/30 Read block 582 len 40960 26/30 Write block 139 len 24576 data: 9 9 10 8 11 9 27/30 Read block 277 len 32768 28/30 Write block 527 len 28672 data: 6 7 7 6 7 9 9 29/30 Read block 18 len 16384 30/30 Read block 561 len 8192 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 4 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:33.019255358Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 90 90 90 170 170 170 T T T 4 080-099 89 89 89 168 168 168 F F F 5 100-119 86 86 86 163 163 163 T T T 6 120-139 89 89 89 168 168 168 T T T 7 140-159 84 84 84 160 160 160 T T T 8 160-179 92 92 92 175 175 175 F F F 9 180-199 92 92 92 175 175 175 F F F 10 200-219 91 91 91 172 172 172 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 92 92 92 175 175 175 F F F 13 260-279 90 90 90 170 170 170 F F F 14 280-299 92 92 92 176 176 176 F F F 15 300-319 88 88 88 166 166 166 F F F 16 320-339 87 87 87 165 165 165 F F F 17 340-359 90 90 90 170 170 170 F F F 18 360-379 92 92 92 174 174 174 F F F 19 380-399 92 92 92 174 174 174 F F F 20 400-419 90 90 90 170 170 170 T T T 21 420-439 89 89 89 168 168 168 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 92 92 92 175 175 175 T T T 24 480-499 84 84 84 159 159 159 F F F 25 500-519 92 92 92 174 174 174 F F F 26 520-539 90 90 90 169 169 169 T T T 27 540-559 85 85 85 161 161 161 F F F 28 560-579 86 86 86 164 164 164 T T T 29 580-599 90 90 90 171 171 171 F F F Max gen: 92, Max flush: 176 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 93 93 93 177 177 177 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 93 93 93 177 177 177 F F F 6 120-139 93 93 93 177 177 177 F F F 7 140-159 93 93 93 177 177 177 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 92 92 92 175 175 175 F F F 10 200-219 91 91 91 172 172 172 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 92 92 92 175 175 175 F F F 13 260-279 90 90 90 170 170 170 F F F 14 280-299 92 92 92 176 176 176 F F F 15 300-319 88 88 88 166 166 166 F F F 16 320-339 87 87 87 165 165 165 F F F 17 340-359 90 90 90 170 170 170 F F F 18 360-379 92 92 92 174 174 174 F F F 19 380-399 92 92 92 174 174 174 F F F 20 400-419 93 93 93 177 177 177 F F F 21 420-439 89 89 89 168 168 168 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 93 93 93 177 177 177 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 92 92 92 174 174 174 F F F 26 520-539 93 93 93 177 177 177 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 93 93 93 177 177 177 F F F 29 580-599 90 90 90 171 171 171 F F F Max gen: 93, Max flush: 177 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.304699801Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.348252043Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.349080822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.34912636Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.34914034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.349154429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} {"msg":"Crucible 37f2207f-bfe8-4852-b749-79a520a55c5b has session id: 41b048bf-29ed-4fc6-ae41-0c6b100c7955","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.349702555Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Upstairs opts: Upstairs UUID: 37f2207f-bfe8-4852-b749-79a520a55c5b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.349750604Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Crucible stats registered with UUID: 37f2207f-bfe8-4852-b749-79a520a55c5b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.349776213Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:37.349793563Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350338468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350373007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} {"msg":"37f2207f-bfe8-4852-b749-79a520a55c5b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350432786Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350460175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"0","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350502804Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"0","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350534513Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"1","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350564882Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"1","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350591852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"2","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350620551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"2","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.35064913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350676479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350703839Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"ds_connection connected from Ok(127.0.0.1:61847)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350731448Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"ds_connection connected from Ok(127.0.0.1:44489)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350761627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"ds_connection connected from Ok(127.0.0.1:56794)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.350849165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.351455919Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"0","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.351483678Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"1","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.352143761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"1","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.35217791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"0","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.352763744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.352797273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.353610222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"2","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354173797Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"2","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[0]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 177, 177, 177, 175, 175, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354734813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[0]R generation[0..12]: [87, 92, 90, 93, 89, 93, 93, 93, 92, 92, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354762992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354779331Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[1]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 177, 177, 177, 175, 175, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354793491Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[1]R generation[0..12]: [87, 92, 90, 93, 89, 93, 93, 93, 92, 92, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354807431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.35482116Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[2]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 177, 177, 177, 175, 175, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.35483496Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[2]R generation[0..12]: [87, 92, 90, 93, 89, 93, 93, 93, 92, 92, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354848119Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354861729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354877699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354891108Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354903768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354930367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"":"downstairs","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354955227Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.354977426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"0","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355001846Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"1","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355025485Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"client":"2","":"downstairs","session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355050084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355063664Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"37f2207f-bfe8-4852-b749-79a520a55c5b is now active with session: 41b048bf-29ed-4fc6-ae41-0c6b100c7955","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355081823Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355102573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568,"session_id":"41b048bf-29ed-4fc6-ae41-0c6b100c7955"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355127312Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} {"msg":"Activated sub_volume 37f2207f-bfe8-4852-b749-79a520a55c5b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:37.355145642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 308 len 8192 data: 8 10 04/30 Read block 388 len 8192 05/30 Read block 578 len 20480 06/30 Write block 192 len 8192 data: 8 5 07/30 Read block 365 len 4096 08/30 Read block 513 len 24576 09/30 Write block 360 len 12288 data: 11 11 13 10/30 Read block 7 len 40960 11/30 Read block 51 len 36864 12/30 Read block 307 len 40960 13/30 Read block 331 len 24576 14/30 Flush 15/30 Write block 320 len 16384 data: 7 7 6 7 16/30 Read block 476 len 28672 17/30 Write block 274 len 28672 data: 11 11 9 9 7 7 6 18/30 Write block 104 len 12288 data: 13 14 11 19/30 Write block 517 len 8192 data: 5 5 20/30 Read block 489 len 28672 21/30 Write block 257 len 8192 data: 6 5 22/30 Flush 23/30 Write block 337 len 24576 data: 9 11 9 12 12 13 24/30 Read block 497 len 12288 25/30 Write block 513 len 32768 data: 9 11 7 7 6 6 6 6 26/30 Write block 572 len 20480 data: 7 7 6 7 7 27/30 Read block 83 len 16384 28/30 Read block 492 len 12288 29/30 Read block 53 len 12288 30/30 Read block 72 len 20480 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:37.374923251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 93 93 93 177 177 177 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 94 94 94 181 181 181 F F F 6 120-139 93 93 93 177 177 177 F F F 7 140-159 93 93 93 177 177 177 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 91 91 91 172 172 172 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 94 94 94 181 181 181 F F F 13 260-279 94 94 94 181 181 181 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 94 94 94 180 180 180 F F F 16 320-339 94 94 94 181 181 181 T T T 17 340-359 90 90 90 170 170 170 T T T 18 360-379 94 94 94 180 180 180 F F F 19 380-399 92 92 92 174 174 174 F F F 20 400-419 93 93 93 177 177 177 F F F 21 420-439 89 89 89 168 168 168 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 93 93 93 177 177 177 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 94 94 94 181 181 181 T T T 26 520-539 93 93 93 177 177 177 T T T 27 540-559 85 85 85 161 161 161 F F F 28 560-579 93 93 93 177 177 177 T T T 29 580-599 90 90 90 171 171 171 F F F Max gen: 94, Max flush: 181 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 93 93 93 177 177 177 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 94 94 94 181 181 181 F F F 6 120-139 93 93 93 177 177 177 F F F 7 140-159 93 93 93 177 177 177 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 91 91 91 172 172 172 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 94 94 94 181 181 181 F F F 13 260-279 94 94 94 181 181 181 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 94 94 94 180 180 180 F F F 16 320-339 95 95 95 182 182 182 F F F 17 340-359 95 95 95 182 182 182 F F F 18 360-379 94 94 94 180 180 180 F F F 19 380-399 92 92 92 174 174 174 F F F 20 400-419 93 93 93 177 177 177 F F F 21 420-439 89 89 89 168 168 168 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 93 93 93 177 177 177 F F F 24 480-499 84 84 84 159 159 159 F F F 25 500-519 95 95 95 182 182 182 F F F 26 520-539 95 95 95 182 182 182 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 95 95 95 182 182 182 F F F 29 580-599 90 90 90 171 171 171 F F F Max gen: 95, Max flush: 182 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.611416233Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.655104133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.65597673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.656031618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.656046098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.656060487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} {"msg":"Crucible 0084dedf-d28c-4d22-a870-49597f76e3de has session id: bf563e20-1de2-4e1c-a858-4f8a4cf9eef5","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.65671432Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Upstairs opts: Upstairs UUID: 0084dedf-d28c-4d22-a870-49597f76e3de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.656763509Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Crucible stats registered with UUID: 0084dedf-d28c-4d22-a870-49597f76e3de","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.656792728Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:51.656814437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657277015Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657320194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} {"msg":"0084dedf-d28c-4d22-a870-49597f76e3de active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657404082Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657429871Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"0","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657454271Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"0","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.65746784Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"1","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.65748234Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"1","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657505469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"2","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657531009Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"2","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657556348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657582117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657610207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35525)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657639996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60191)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.657935548Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"ds_connection connected from Ok(127.0.0.1:49225)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.658002186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.658492243Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"0","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.658520583Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"1","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.659181845Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"0","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.659317742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"1","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.6597621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.659885427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.659965905Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"2","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.660559359Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"2","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[0]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 181, 177, 177, 175, 180, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661113524Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[0]R generation[0..12]: [87, 92, 90, 93, 89, 94, 93, 93, 92, 94, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661137414Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661152163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[1]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 181, 177, 177, 175, 180, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661166153Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[1]R generation[0..12]: [87, 92, 90, 93, 89, 94, 93, 93, 92, 94, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661183672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661197872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[2]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 181, 177, 177, 175, 180, 172, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661211522Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[2]R generation[0..12]: [87, 92, 90, 93, 89, 94, 93, 93, 92, 94, 91, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661224921Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661238681Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.66125813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661305289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661322659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661335568Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"":"downstairs","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661349098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661369608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"0","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661394317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"1","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661417006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"client":"2","":"downstairs","session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661439626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661461015Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"0084dedf-d28c-4d22-a870-49597f76e3de is now active with session: bf563e20-1de2-4e1c-a858-4f8a4cf9eef5","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-19T17:04:51.661487484Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661509684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576,"session_id":"bf563e20-1de2-4e1c-a858-4f8a4cf9eef5"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661523164Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} {"msg":"Activated sub_volume 0084dedf-d28c-4d22-a870-49597f76e3de","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:51.661534513Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 138 len 8192 data: 8 10 02/30 Write block 213 len 28672 data: 6 6 6 5 5 5 4 03/30 Write block 125 len 36864 data: 6 3 6 6 6 4 4 7 8 04/30 Write block 418 len 32768 data: 10 12 12 9 9 10 10 9 05/30 Read block 306 len 20480 06/30 Write block 141 len 32768 data: 11 9 12 10 11 11 11 7 07/30 Flush 08/30 Write block 111 len 28672 data: 7 5 5 7 6 5 7 09/30 Write block 301 len 24576 data: 4 5 5 7 8 8 10/30 Read block 105 len 28672 11/30 Read block 301 len 40960 12/30 Write block 420 len 4096 data: 13 13/30 Write block 494 len 20480 data: 9 10 10 8 7 14/30 Write block 571 len 28672 data: 6 8 8 7 8 8 8 15/30 Read block 303 len 24576 16/30 Read block 444 len 40960 17/30 Flush 18/30 Write block 131 len 8192 data: 5 8 19/30 Read block 416 len 28672 20/30 Read block 373 len 24576 21/30 Write block 363 len 40960 data: 11 8 7 8 9 9 6 8 8 9 22/30 Write block 331 len 12288 data: 8 8 10 23/30 Read block 359 len 20480 24/30 Read block 65 len 16384 25/30 Write block 578 len 20480 data: 9 8 7 7 7 26/30 Read block 488 len 4096 27/30 Read block 463 len 12288 28/30 Write block 561 len 24576 data: 3 6 8 10 10 10 29/30 Read block 5 len 8192 30/30 Write block 495 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 2 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:51.681559505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 93 93 93 177 177 177 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 96 96 96 184 184 184 F F F 6 120-139 96 96 96 183 183 183 T T T 7 140-159 96 96 96 183 183 183 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 94 94 94 181 181 181 F F F 13 260-279 94 94 94 181 181 181 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 96 96 96 184 184 184 F F F 16 320-339 95 95 95 182 182 182 T T T 17 340-359 95 95 95 182 182 182 F F F 18 360-379 94 94 94 180 180 180 T T T 19 380-399 92 92 92 174 174 174 F F F 20 400-419 96 96 96 183 183 183 F F F 21 420-439 96 96 96 184 184 184 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 93 93 93 177 177 177 F F F 24 480-499 96 96 96 184 184 184 T T T 25 500-519 95 95 95 182 182 182 F F F 26 520-539 95 95 95 182 182 182 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 96 96 96 184 184 184 T T T 29 580-599 90 90 90 171 171 171 T T T Max gen: 96, Max flush: 184 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 93 93 93 177 177 177 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 96 96 96 184 184 184 F F F 6 120-139 97 97 97 185 185 185 F F F 7 140-159 96 96 96 183 183 183 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 88 88 88 167 167 167 F F F 12 240-259 94 94 94 181 181 181 F F F 13 260-279 94 94 94 181 181 181 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 96 96 96 184 184 184 F F F 16 320-339 97 97 97 185 185 185 F F F 17 340-359 95 95 95 182 182 182 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 92 92 92 174 174 174 F F F 20 400-419 96 96 96 183 183 183 F F F 21 420-439 96 96 96 184 184 184 F F F 22 440-459 89 89 89 168 168 168 F F F 23 460-479 93 93 93 177 177 177 F F F 24 480-499 97 97 97 185 185 185 F F F 25 500-519 95 95 95 182 182 182 F F F 26 520-539 95 95 95 182 182 182 F F F 27 540-559 85 85 85 161 161 161 F F F 28 560-579 97 97 97 185 185 185 F F F 29 580-599 97 97 97 185 185 185 F F F Max gen: 97, Max flush: 185 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.924800408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.969308965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.970020957Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.970056336Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.970070255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.970084795Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} {"msg":"Crucible 9f0560e8-6ea5-4fbf-8a29-1ddbc7367f65 has session id: 8fa230e9-b92d-4d1c-b781-e550f61b01e6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.97065818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Upstairs opts: Upstairs UUID: 9f0560e8-6ea5-4fbf-8a29-1ddbc7367f65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.970701039Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Crucible stats registered with UUID: 9f0560e8-6ea5-4fbf-8a29-1ddbc7367f65","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.970726448Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:55.970748337Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971146927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971171456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} {"msg":"9f0560e8-6ea5-4fbf-8a29-1ddbc7367f65 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971232194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971249094Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"0","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971271713Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"0","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971285333Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"1","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971303073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"1","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971316472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"2","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971330252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"2","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971343151Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971356801Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971369941Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51641)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971448449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49651)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971482228Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49530)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.971507037Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.972321596Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"0","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.972374315Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"1","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.973065296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"0","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.973222872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"1","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.973262431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"2","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.973614912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.973832976Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.973913144Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"2","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[0]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 184, 185, 183, 175, 180, 183, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.97444107Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[0]R generation[0..12]: [87, 92, 90, 93, 89, 96, 97, 96, 92, 94, 96, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974464339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974478959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[1]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 184, 185, 183, 175, 180, 183, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974492569Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[1]R generation[0..12]: [87, 92, 90, 93, 89, 96, 97, 96, 92, 94, 96, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974505908Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974519458Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[2]R flush_numbers[0..12]: [165, 176, 169, 177, 168, 184, 185, 183, 175, 180, 183, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974532838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[2]R generation[0..12]: [87, 92, 90, 93, 89, 96, 97, 96, 92, 94, 96, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974546127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974559567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974572146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974584736Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974600296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974613285Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"":"downstairs","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974626285Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974637855Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"0","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974650884Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":Wait for a query_work_queue command to finish before sending IO 1584,"client":"1","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974672404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"client":"2","":"downstairs","session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974685423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974697343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"9f0560e8-6ea5-4fbf-8a29-1ddbc7367f65 is now active with session: 8fa230e9-b92d-4d1c-b781-e550f61b01e6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974709703Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974721322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584,"session_id":"8fa230e9-b92d-4d1c-b781-e550f61b01e6"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974733502Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} {"msg":"Activated sub_volume 9f0560e8-6ea5-4fbf-8a29-1ddbc7367f65","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:04:55.974744642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 574 len 36864 data: 8 9 9 9 10 9 8 8 8 02/30 Read block 165 len 4096 03/30 Write block 106 len 24576 data: 12 10 9 9 8 8 04/30 Read block 173 len 20480 05/30 Read block 578 len 20480 06/30 Write block 535 len 28672 data: 10 11 11 11 12 10 9 07/30 Write block 470 len 8192 data: 5 4 08/30 Write block 230 len 12288 data: 6 6 4 09/30 Read block 500 len 40960 10/30 Write block 71 len 36864 data: 7 5 5 8 10 8 8 8 8 11/30 Read block 396 len 36864 12/30 Read block 18 len 24576 13/30 Read block 352 len 28672 14/30 Read block 439 len 12288 15/30 Read block 377 len 16384 16/30 Write block 428 len 40960 data: 7 7 8 10 6 5 6 7 7 7 17/30 Write block 111 len 36864 data: 9 6 6 8 7 6 8 8 8 18/30 Read block 502 len 32768 19/30 Read block 112 len 8192 20/30 Read block 386 len 36864 21/30 Write block 221 len 24576 data: 4 5 6 10 9 11 22/30 Write block 407 len 16384 data: 10 10 7 9 23/30 Read block 542 len 32768 24/30 Read block 292 len 20480 25/30 Read block 573 len 40960 26/30 Flush 27/30 Write block 452 len 16384 data: 8 8 8 8 28/30 Write block 477 len 24576 data: 7 8 10 11 9 7 29/30 Write block 453 len 40960 data: 9 9 9 7 7 8 7 5 6 6 30/30 Read block 317 len 28672 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:04:55.99722708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 98 98 98 186 186 186 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 98 98 98 186 186 186 F F F 6 120-139 97 97 97 185 185 185 F F F 7 140-159 96 96 96 183 183 183 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 94 94 94 181 181 181 F F F 13 260-279 94 94 94 181 181 181 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 96 96 96 184 184 184 F F F 16 320-339 97 97 97 185 185 185 F F F 17 340-359 95 95 95 182 182 182 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 92 92 92 174 174 174 F F F 20 400-419 98 98 98 186 186 186 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 89 89 89 168 168 168 T T T 23 460-479 98 98 98 186 186 186 T T T 24 480-499 97 97 97 185 185 185 T T T 25 500-519 95 95 95 182 182 182 F F F 26 520-539 98 98 98 186 186 186 F F F 27 540-559 98 98 98 186 186 186 F F F 28 560-579 98 98 98 186 186 186 F F F 29 580-599 98 98 98 186 186 186 F F F Max gen: 98, Max flush: 186 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 F F F 1 020-039 92 92 92 176 176 176 F F F 2 040-059 90 90 90 169 169 169 F F F 3 060-079 98 98 98 186 186 186 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 98 98 98 186 186 186 F F F 6 120-139 97 97 97 185 185 185 F F F 7 140-159 96 96 96 183 183 183 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 94 94 94 181 181 181 F F F 13 260-279 94 94 94 181 181 181 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 96 96 96 184 184 184 F F F 16 320-339 97 97 97 185 185 185 F F F 17 340-359 95 95 95 182 182 182 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 92 92 92 174 174 174 F F F 20 400-419 98 98 98 186 186 186 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 99 99 99 187 187 187 F F F 23 460-479 99 99 99 187 187 187 F F F 24 480-499 99 99 99 187 187 187 F F F 25 500-519 95 95 95 182 182 182 F F F 26 520-539 98 98 98 186 186 186 F F F 27 540-559 98 98 98 186 186 186 F F F 28 560-579 98 98 98 186 186 186 F F F 29 580-599 98 98 98 186 186 186 F F F Max gen: 99, Max flush: 187 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.226593208Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.271452687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.272367962Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.272405532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.272418971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.272432831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} {"msg":"Crucible 1dee1811-bb87-4a07-bd67-e01cd56c9cf0 has session id: 0adb00f1-6232-4a51-b55e-5581d5a3b927","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273137102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Upstairs opts: Upstairs UUID: 1dee1811-bb87-4a07-bd67-e01cd56c9cf0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273176011Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Crucible stats registered with UUID: 1dee1811-bb87-4a07-bd67-e01cd56c9cf0","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273191831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:00.27320448Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273635369Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273659519Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} {"msg":"1dee1811-bb87-4a07-bd67-e01cd56c9cf0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273718837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273735906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"0","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273758176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"0","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273771636Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"1","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273785645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"1","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273799105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"2","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273867153Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"2","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273889142Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273914932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.273940261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"ds_connection connected from Ok(127.0.0.1:52577)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.27396452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"ds_connection connected from Ok(127.0.0.1:43928)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.27398773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"ds_connection connected from Ok(127.0.0.1:47901)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.274067458Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.274632303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"1","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.274674182Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"0","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.275247547Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"1","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.275401193Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"0","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.275785452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.275918089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.276042806Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"2","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.276550252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"2","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[0]R flush_numbers[0..12]: [165, 176, 169, 186, 168, 186, 185, 183, 175, 180, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277082968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[0]R generation[0..12]: [87, 92, 90, 98, 89, 98, 97, 96, 92, 94, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277103258Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277117868Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[1]R flush_numbers[0..12]: [165, 176, 169, 186, 168, 186, 185, 183, 175, 180, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277131517Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[1]R generation[0..12]: [87, 92, 90, 98, 89, 98, 97, 96, 92, 94, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277145097Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277158866Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[2]R flush_numbers[0..12]: [165, 176, 169, 186, 168, 186, 185, 183, 175, 180, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277176266Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[2]R generation[0..12]: [87, 92, 90, 98, 89, 98, 97, 96, 92, 94, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277189886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277203495Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277216065Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277228934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277241454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277254034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"":"downstairs","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277267313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277279133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"0","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277299573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"1","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277312662Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"client":"2","":"downstairs","session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277325692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277337742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"1dee1811-bb87-4a07-bd67-e01cd56c9cf0 is now active with session: 0adb00f1-6232-4a51-b55e-5581d5a3b927","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277350211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277361741Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592,"session_id":"0adb00f1-6232-4a51-b55e-5581d5a3b927"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.277375551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} {"msg":"Activated sub_volume 1dee1811-bb87-4a07-bd67-e01cd56c9cf0","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:00.2773868Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 408 len 28672 data: 11 8 10 9 8 9 6 02/30 Flush 03/30 Write block 38 len 32768 data: 5 6 7 7 8 9 9 8 04/30 Read block 351 len 32768 05/30 Read block 445 len 4096 06/30 Flush 07/30 Write block 266 len 24576 data: 9 10 11 11 12 8 08/30 Write block 256 len 8192 data: 5 7 09/30 Read block 254 len 4096 10/30 Write block 119 len 16384 data: 9 8 7 6 11/30 Read block 355 len 24576 12/30 Write block 548 len 16384 data: 15 13 13 13 13/30 Read block 283 len 36864 14/30 Write block 496 len 8192 data: 11 9 15/30 Write block 9 len 40960 data: 6 7 6 6 6 7 8 7 8 10 16/30 Write block 515 len 4096 data: 8 17/30 Read block 95 len 28672 18/30 Read block 504 len 16384 19/30 Write block 303 len 36864 data: 6 8 9 9 8 9 11 10 9 20/30 Write block 152 len 12288 data: 5 4 4 21/30 Write block 398 len 32768 data: 13 11 12 11 9 10 9 9 22/30 Write block 256 len 28672 data: 6 8 6 9 9 12 13 23/30 Read block 405 len 36864 24/30 Write block 261 len 28672 data: 13 14 11 10 9 10 11 25/30 Read block 197 len 4096 26/30 Write block 516 len 16384 data: 8 7 7 7 27/30 Read block 307 len 8192 28/30 Write block 591 len 28672 data: 5 5 5 6 6 6 3 29/30 Write block 569 len 36864 data: 9 9 7 9 9 9 10 10 10 30/30 Write block 245 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:00.29675857Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 165 165 165 T T T 1 020-039 100 100 100 189 189 189 F F F 2 040-059 100 100 100 189 189 189 F F F 3 060-079 98 98 98 186 186 186 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 98 98 98 186 186 186 T T T 6 120-139 97 97 97 185 185 185 T T T 7 140-159 96 96 96 183 183 183 T T T 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 94 94 94 181 181 181 T T T 13 260-279 94 94 94 181 181 181 T T T 14 280-299 94 94 94 181 181 181 F F F 15 300-319 96 96 96 184 184 184 T T T 16 320-339 97 97 97 185 185 185 F F F 17 340-359 95 95 95 182 182 182 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 92 92 92 174 174 174 T T T 20 400-419 100 100 100 188 188 188 T T T 21 420-439 98 98 98 186 186 186 F F F 22 440-459 99 99 99 187 187 187 F F F 23 460-479 99 99 99 187 187 187 F F F 24 480-499 99 99 99 187 187 187 T T T 25 500-519 95 95 95 182 182 182 T T T 26 520-539 98 98 98 186 186 186 F F F 27 540-559 98 98 98 186 186 186 T T T 28 560-579 98 98 98 186 186 186 T T T 29 580-599 98 98 98 186 186 186 T T T Max gen: 100, Max flush: 189 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 190 190 190 F F F 1 020-039 100 100 100 189 189 189 F F F 2 040-059 100 100 100 189 189 189 F F F 3 060-079 98 98 98 186 186 186 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 101 101 101 190 190 190 F F F 6 120-139 101 101 101 190 190 190 F F F 7 140-159 101 101 101 190 190 190 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 94 94 94 180 180 180 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 101 101 101 190 190 190 F F F 13 260-279 101 101 101 190 190 190 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 101 101 101 190 190 190 F F F 16 320-339 97 97 97 185 185 185 F F F 17 340-359 95 95 95 182 182 182 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 101 101 101 190 190 190 F F F 20 400-419 101 101 101 190 190 190 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 99 99 99 187 187 187 F F F 23 460-479 99 99 99 187 187 187 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 101 101 101 190 190 190 F F F 26 520-539 98 98 98 186 186 186 F F F 27 540-559 101 101 101 190 190 190 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 101 101 101 190 190 190 F F F Max gen: 101, Max flush: 190 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.6028546Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.646485541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.647308209Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.647350598Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.647366727Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.647381227Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} {"msg":"Crucible 37178bdf-1531-4f70-b83c-327ffc00360a has session id: ac5f3d6e-021a-4ed3-ba8f-0695c0311848","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.647987821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Upstairs opts: Upstairs UUID: 37178bdf-1531-4f70-b83c-327ffc00360a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648048459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Crucible stats registered with UUID: 37178bdf-1531-4f70-b83c-327ffc00360a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648074399Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:14.648089538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648556396Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648580345Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} {"msg":"37178bdf-1531-4f70-b83c-327ffc00360a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648643714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648661003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"0","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648685863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"0","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648717182Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"1","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648745781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"1","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.64877329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"2","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.64880501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"2","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648832999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648863588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.648953286Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"ds_connection connected from Ok(127.0.0.1:55266)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.649037613Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"ds_connection connected from Ok(127.0.0.1:43022)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.64915008Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"ds_connection connected from Ok(127.0.0.1:44822)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.649193359Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.649801913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"0","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.649957909Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"2","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.650384668Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"0","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.650519714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"2","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.651009401Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.651120669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.651507858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"1","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652026075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"1","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[0]R flush_numbers[0..12]: [190, 189, 189, 186, 168, 190, 190, 190, 175, 180, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652531061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[0]R generation[0..12]: [101, 100, 100, 98, 89, 101, 101, 101, 92, 94, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652555091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.65256976Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[1]R flush_numbers[0..12]: [190, 189, 189, 186, 168, 190, 190, 190, 175, 180, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.65258383Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[1]R generation[0..12]: [101, 100, 100, 98, 89, 101, 101, 101, 92, 94, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.65259737Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652611029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[2]R flush_numbers[0..12]: [190, 189, 189, 186, 168, 190, 190, 190, 175, 180, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652624409Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[2]R generation[0..12]: [101, 100, 100, 98, 89, 101, 101, 101, 92, 94, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652637578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652651178Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652663918Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652676817Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652751255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652769835Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"":"downstairs","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652782945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652794754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"0","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652807504Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"1","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652820364Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"client":"2","":"downstairs","session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652833163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652845063Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"37178bdf-1531-4f70-b83c-327ffc00360a is now active with session: ac5f3d6e-021a-4ed3-ba8f-0695c0311848","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652857783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652869463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600,"session_id":"ac5f3d6e-021a-4ed3-ba8f-0695c0311848"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652881372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} {"msg":"Activated sub_volume 37178bdf-1531-4f70-b83c-327ffc00360a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:14.652892602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 543 len 28672 data: 13 12 13 12 16 16 14 02/30 Read block 574 len 8192 03/30 Write block 583 len 12288 data: 7 8 8 04/30 Read block 115 len 8192 05/30 Read block 203 len 16384 06/30 Read block 568 len 36864 07/30 Write block 119 len 40960 data: 10 9 8 7 6 6 7 4 7 7 08/30 Write block 307 len 12288 data: 9 10 12 09/30 Read block 433 len 12288 10/30 Write block 45 len 28672 data: 9 5 6 6 6 7 7 11/30 Flush 12/30 Write block 252 len 24576 data: 10 7 7 6 7 9 13/30 Read block 250 len 28672 14/30 Read block 29 len 12288 15/30 Read block 154 len 32768 16/30 Read block 542 len 24576 17/30 Write block 518 len 8192 data: 8 8 18/30 Write block 31 len 20480 data: 11 10 9 8 9 19/30 Write block 193 len 28672 data: 6 5 5 6 4 5 6 20/30 Flush 21/30 Write block 62 len 32768 data: 9 9 9 9 10 9 9 10 22/30 Read block 448 len 4096 23/30 Read block 479 len 24576 24/30 Read block 98 len 36864 25/30 Read block 129 len 32768 26/30 Read block 143 len 20480 27/30 Write block 337 len 8192 data: 10 12 28/30 Write block 142 len 32768 data: 10 13 11 12 12 12 8 6 29/30 Read block 88 len 40960 30/30 Read block 212 len 20480 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:14.67458747Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 190 190 190 F F F 1 020-039 102 102 102 192 192 192 F F F 2 040-059 102 102 102 191 191 191 F F F 3 060-079 98 98 98 186 186 186 T T T 4 080-099 89 89 89 168 168 168 F F F 5 100-119 102 102 102 191 191 191 F F F 6 120-139 102 102 102 191 191 191 F F F 7 140-159 101 101 101 190 190 190 T T T 8 160-179 92 92 92 175 175 175 F F F 9 180-199 102 102 102 192 192 192 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 102 102 102 192 192 192 F F F 13 260-279 101 101 101 190 190 190 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 102 102 102 191 191 191 F F F 16 320-339 97 97 97 185 185 185 T T T 17 340-359 95 95 95 182 182 182 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 101 101 101 190 190 190 F F F 20 400-419 101 101 101 190 190 190 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 99 99 99 187 187 187 F F F 23 460-479 99 99 99 187 187 187 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 98 98 98 186 186 186 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 102 102 102 191 191 191 F F F Max gen: 102, Max flush: 192 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 190 190 190 F F F 1 020-039 102 102 102 192 192 192 F F F 2 040-059 102 102 102 191 191 191 F F F 3 060-079 103 103 103 193 193 193 F F F 4 080-099 89 89 89 168 168 168 F F F 5 100-119 102 102 102 191 191 191 F F F 6 120-139 102 102 102 191 191 191 F F F 7 140-159 103 103 103 193 193 193 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 102 102 102 192 192 192 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 102 102 102 192 192 192 F F F 13 260-279 101 101 101 190 190 190 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 102 102 102 191 191 191 F F F 16 320-339 103 103 103 193 193 193 F F F 17 340-359 95 95 95 182 182 182 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 101 101 101 190 190 190 F F F 20 400-419 101 101 101 190 190 190 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 99 99 99 187 187 187 F F F 23 460-479 99 99 99 187 187 187 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 98 98 98 186 186 186 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 102 102 102 191 191 191 F F F Max gen: 103, Max flush: 193 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.902207945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.945906094Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.946748162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.946785621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.9468002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.94682149Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} {"msg":"Crucible aaa8fb52-be07-4ee3-9a0f-20a5c03efa0d has session id: 0db4fea9-002c-406e-a423-a97887e0d9fc","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.947479822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Upstairs opts: Upstairs UUID: aaa8fb52-be07-4ee3-9a0f-20a5c03efa0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.947524761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Crucible stats registered with UUID: aaa8fb52-be07-4ee3-9a0f-20a5c03efa0d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.94755172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:18.94757136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.947927231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.94795557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} {"msg":"aaa8fb52-be07-4ee3-9a0f-20a5c03efa0d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948032858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948059187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"0","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948083256Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"0","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948097296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"1","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948111336Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"1","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948126575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948151634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948177494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948203773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948228892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:18.948537594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37343)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948583393Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54079)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948611242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:18.948636652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:18.948661451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.948681161Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.94909336Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"0","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.949122699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"1","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.949880969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"0","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.949908568Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"1","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.950508952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:18.950533382Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.950770658Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41489)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.950853815Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.951783601Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.952309587Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[0]R flush_numbers[0..12]: [190, 192, 191, 193, 168, 191, 191, 193, 175, 192, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.952870842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[0]R generation[0..12]: [101, 102, 102, 103, 89, 102, 102, 103, 92, 102, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.952899532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.952916301Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[1]R flush_numbers[0..12]: [190, 192, 191, 193, 168, 191, 191, 193, 175, 192, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.952930671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[1]R generation[0..12]: [101, 102, 102, 103, 89, 102, 102, 103, 92, 102, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.95294444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.9529583Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[2]R flush_numbers[0..12]: [190, 192, 191, 193, 168, 191, 191, 193, 175, 192, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.95297191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[2]R generation[0..12]: [101, 102, 102, 103, 89, 102, 102, 103, 92, 102, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.952985169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.952998759Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953018618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953042928Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953067417Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953090837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"":"downstairs","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953112326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:05:28.953139775Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"0","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953161205Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"1","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953174734Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"client":"2","":"downstairs","session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953189054Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953210073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"aaa8fb52-be07-4ee3-9a0f-20a5c03efa0d is now active with session: 0db4fea9-002c-406e-a423-a97887e0d9fc","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953234203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953256392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608,"session_id":"0db4fea9-002c-406e-a423-a97887e0d9fc"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953279912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} {"msg":"Activated sub_volume aaa8fb52-be07-4ee3-9a0f-20a5c03efa0d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:28.953301971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 28672 data: 12 11 10 9 10 9 8 02/30 Read block 473 len 28672 03/30 Read block 427 len 24576 04/30 Read block 581 len 16384 05/30 Read block 257 len 24576 06/30 Write block 87 len 4096 data: 9 07/30 Read block 139 len 32768 08/30 Read block 199 len 8192 09/30 Read block 224 len 16384 10/30 Write block 158 len 8192 data: 7 6 11/30 Flush 12/30 Write block 475 len 12288 data: 9 7 8 13/30 Read block 492 len 8192 14/30 Read block 46 len 28672 15/30 Write block 29 len 40960 data: 12 12 13 12 11 10 11 10 9 6 16/30 Write block 127 len 20480 data: 8 8 7 5 6 17/30 Read block 46 len 36864 18/30 Read block 545 len 40960 19/30 Read block 439 len 16384 20/30 Write block 75 len 16384 data: 11 9 9 9 21/30 Write block 388 len 32768 data: 3 3 4 8 9 9 13 13 22/30 Read block 6 len 32768 23/30 Read block 37 len 32768 24/30 Write block 350 len 12288 data: 8 8 6 25/30 Write block 82 len 4096 data: 9 26/30 Read block 400 len 40960 27/30 Write block 534 len 12288 data: 9 11 12 28/30 Read block 280 len 16384 29/30 Write block 105 len 20480 data: 15 13 11 10 10 30/30 Write block 527 len 20480 data: 7 8 8 7 8 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:28.974184511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 190 190 190 F F F 1 020-039 104 104 104 194 194 194 T T T 2 040-059 102 102 102 191 191 191 F F F 3 060-079 103 103 103 193 193 193 T T T 4 080-099 104 104 104 194 194 194 T T T 5 100-119 102 102 102 191 191 191 T T T 6 120-139 102 102 102 191 191 191 T T T 7 140-159 104 104 104 194 194 194 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 102 102 102 192 192 192 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 102 102 102 192 192 192 F F F 13 260-279 101 101 101 190 190 190 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 102 102 102 191 191 191 F F F 16 320-339 103 103 103 193 193 193 F F F 17 340-359 95 95 95 182 182 182 T T T 18 360-379 97 97 97 185 185 185 F F F 19 380-399 101 101 101 190 190 190 T T T 20 400-419 101 101 101 190 190 190 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 99 99 99 187 187 187 F F F 23 460-479 99 99 99 187 187 187 T T T 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 98 98 98 186 186 186 T T T 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 102 102 102 191 191 191 F F F Max gen: 104, Max flush: 194 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 190 190 190 F F F 1 020-039 105 105 105 195 195 195 F F F 2 040-059 102 102 102 191 191 191 F F F 3 060-079 105 105 105 195 195 195 F F F 4 080-099 105 105 105 195 195 195 F F F 5 100-119 105 105 105 195 195 195 F F F 6 120-139 105 105 105 195 195 195 F F F 7 140-159 104 104 104 194 194 194 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 102 102 102 192 192 192 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 102 102 102 192 192 192 F F F 13 260-279 101 101 101 190 190 190 F F F 14 280-299 94 94 94 181 181 181 F F F 15 300-319 102 102 102 191 191 191 F F F 16 320-339 103 103 103 193 193 193 F F F 17 340-359 105 105 105 195 195 195 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 105 105 105 195 195 195 F F F 20 400-419 101 101 101 190 190 190 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 99 99 99 187 187 187 F F F 23 460-479 105 105 105 195 195 195 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 105 105 105 195 195 195 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 102 102 102 191 191 191 F F F Max gen: 105, Max flush: 195 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.234150723Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.277689536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.278414557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.278457086Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.278471396Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.278485625Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} {"msg":"Crucible 46102424-5fbf-4e03-a4a4-7a7fe82f06db has session id: 29597008-487e-4458-8595-fa3ec02ea077","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279144788Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Upstairs opts: Upstairs UUID: 46102424-5fbf-4e03-a4a4-7a7fe82f06db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279179967Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Crucible stats registered with UUID: 46102424-5fbf-4e03-a4a4-7a7fe82f06db","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279195566Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:33.279212456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279797851Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.2798311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} {"msg":"46102424-5fbf-4e03-a4a4-7a7fe82f06db active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279892808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279920018Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"0","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279955297Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"0","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.279986846Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280020655Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280047634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"2","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280077373Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"2","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280102073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280127162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280152091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"ds_connection connected from Ok(127.0.0.1:58401)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280175761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"ds_connection connected from Ok(127.0.0.1:45035)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.28019852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:33.280226729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:33.280335527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:33.280371186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280474963Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.280923651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"0","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.28095187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"2","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.281596463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"0","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.281628233Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"2","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.282239126Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:33.282265876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.281747661Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"ds_connection connected from Ok(127.0.0.1:41103)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.281953146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.282956869Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.283634552Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[0]R flush_numbers[0..12]: [190, 195, 191, 195, 195, 195, 195, 194, 175, 192, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284256925Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[0]R generation[0..12]: [101, 105, 102, 105, 105, 105, 105, 104, 92, 102, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284290524Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284307874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[1]R flush_numbers[0..12]: [190, 195, 191, 195, 195, 195, 195, 194, 175, 192, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284322183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[1]R generation[0..12]: [101, 105, 102, 105, 105, 105, 105, 104, 92, 102, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284335703Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284349893Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[2]R flush_numbers[0..12]: [190, 195, 191, 195, 195, 195, 195, 194, 175, 192, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284363402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[2]R generation[0..12]: [101, 105, 102, 105, 105, 105, 105, 104, 92, 102, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284377052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284391222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284404251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284417431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284430111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.28444302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"":"downstairs","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.28445605Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.28446783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"0","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284482959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"1","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284496049Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"client":"2","":"downstairs","session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284509138Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284520958Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"46102424-5fbf-4e03-a4a4-7a7fe82f06db is now active with session: 29597008-487e-4458-8595-fa3ec02ea077","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284535718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"Set Active after no reconciliation","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284555147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616,"session_id":"29597008-487e-4458-8595-fa3ec02ea077"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284570187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} {"msg":"Activated sub_volume 46102424-5fbf-4e03-a4a4-7a7fe82f06db","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:43.284583856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 12288 data: 5 5 4 02/30 Write block 325 len 8192 data: 6 8 03/30 Read block 196 len 24576 04/30 Read block 210 len 40960 05/30 Read block 301 len 36864 06/30 Flush 07/30 Write block 136 len 36864 data: 7 8 9 11 10 12 11 14 12 08/30 Write block 59 len 40960 data: 7 7 7 10 10 10 10 11 10 10 09/30 Write block 450 len 4096 data: 6 10/30 Write block 289 len 20480 data: 4 6 6 5 7 11/30 Read block 437 len 36864 12/30 Write block 9 len 32768 data: 7 8 7 7 7 8 9 8 13/30 Read block 286 len 16384 14/30 Read block 463 len 40960 15/30 Read block 32 len 32768 16/30 Flush 17/30 Write block 589 len 16384 data: 5 7 6 6 18/30 Write block 6 len 24576 data: 7 7 5 8 9 8 19/30 Read block 344 len 20480 20/30 Read block 139 len 28672 21/30 Read block 282 len 36864 22/30 Read block 552 len 4096 23/30 Read block 580 len 8192 24/30 Write block 184 len 16384 data: 9 7 6 6 25/30 Read block 226 len 4096 26/30 Flush 27/30 Write block 15 len 36864 data: 10 9 9 11 10 9 9 9 7 28/30 Write block 302 len 40960 data: 6 7 9 10 10 10 11 13 11 10 29/30 Write block 27 len 36864 data: 10 12 13 13 14 13 12 11 12 30/30 Write block 266 len 28672 data: 11 12 12 12 13 9 8 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:43.305158765Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 198 198 198 T T T 1 020-039 105 105 105 195 195 195 T T T 2 040-059 106 106 106 197 197 197 F F F 3 060-079 106 106 106 197 197 197 F F F 4 080-099 105 105 105 195 195 195 F F F 5 100-119 105 105 105 195 195 195 F F F 6 120-139 106 106 106 197 197 197 F F F 7 140-159 106 106 106 197 197 197 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 106 106 106 198 198 198 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 102 102 102 192 192 192 F F F 13 260-279 101 101 101 190 190 190 T T T 14 280-299 106 106 106 197 197 197 F F F 15 300-319 102 102 102 191 191 191 T T T 16 320-339 106 106 106 196 196 196 F F F 17 340-359 105 105 105 195 195 195 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 105 105 105 195 195 195 F F F 20 400-419 101 101 101 190 190 190 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 105 105 105 195 195 195 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 105 105 105 195 195 195 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 106, Max flush: 198 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 199 199 199 F F F 1 020-039 107 107 107 199 199 199 F F F 2 040-059 106 106 106 197 197 197 F F F 3 060-079 106 106 106 197 197 197 F F F 4 080-099 105 105 105 195 195 195 F F F 5 100-119 105 105 105 195 195 195 F F F 6 120-139 106 106 106 197 197 197 F F F 7 140-159 106 106 106 197 197 197 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 106 106 106 198 198 198 F F F 10 200-219 96 96 96 183 183 183 F F F 11 220-239 98 98 98 186 186 186 F F F 12 240-259 102 102 102 192 192 192 F F F 13 260-279 107 107 107 199 199 199 F F F 14 280-299 106 106 106 197 197 197 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 106 106 106 196 196 196 F F F 17 340-359 105 105 105 195 195 195 F F F 18 360-379 97 97 97 185 185 185 F F F 19 380-399 105 105 105 195 195 195 F F F 20 400-419 101 101 101 190 190 190 F F F 21 420-439 98 98 98 186 186 186 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 105 105 105 195 195 195 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 105 105 105 195 195 195 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 107, Max flush: 199 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.53997605Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.584096188Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.585048492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.585087371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.585101481Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.585115781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} {"msg":"Crucible 734fb182-a4d8-4ad6-abb0-4ca5e9460be6 has session id: f8c012da-5cbf-4962-a37f-742773b81f11","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.585708955Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Upstairs opts: Upstairs UUID: 734fb182-a4d8-4ad6-abb0-4ca5e9460be6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.585754744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Crucible stats registered with UUID: 734fb182-a4d8-4ad6-abb0-4ca5e9460be6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.585778783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:57.585799152Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.58628194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586309679Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} {"msg":"734fb182-a4d8-4ad6-abb0-4ca5e9460be6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586367248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586394117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"0","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586426846Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"0","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586453515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"1","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586479975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"1","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586505554Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"2","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586536573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"2","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586561992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586588512Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586616581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"ds_connection connected from Ok(127.0.0.1:43768)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586679729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"ds_connection connected from Ok(127.0.0.1:56782)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586707889Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"ds_connection connected from Ok(127.0.0.1:60816)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.586796836Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.587373051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"2","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.587412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"0","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.58816494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"2","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.588199779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"0","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.588813813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.588911721Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.589092006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"1","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.589626162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"1","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[0]R flush_numbers[0..12]: [199, 199, 197, 197, 195, 195, 197, 197, 175, 198, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590167337Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[0]R generation[0..12]: [107, 107, 106, 106, 105, 105, 106, 106, 92, 106, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590191487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590206476Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[1]R flush_numbers[0..12]: [199, 199, 197, 197, 195, 195, 197, 197, 175, 198, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590220846Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[1]R generation[0..12]: [107, 107, 106, 106, 105, 105, 106, 106, 92, 106, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590234846Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590248515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[2]R flush_numbers[0..12]: [199, 199, 197, 197, 195, 195, 197, 197, 175, 198, 183, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590262245Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[2]R generation[0..12]: [107, 107, 106, 106, 105, 105, 106, 106, 92, 106, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590275915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590289634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590306754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590327963Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590341853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590354472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"":"downstairs","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:05:57.590367382Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590390442Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"0","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590403441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"1","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590416361Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"client":"2","":"downstairs","session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.59042923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.5904412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"734fb182-a4d8-4ad6-abb0-4ca5e9460be6 is now active with session: f8c012da-5cbf-4962-a37f-742773b81f11","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.59046192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590474019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624,"session_id":"f8c012da-5cbf-4962-a37f-742773b81f11"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590485879Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} {"msg":"Activated sub_volume 734fb182-a4d8-4ad6-abb0-4ca5e9460be6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:05:57.590496729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 367 len 36864 data: 10 10 7 9 9 10 10 12 9 02/30 Write block 96 len 40960 data: 12 12 11 11 12 12 11 13 14 16 03/30 Read block 15 len 16384 04/30 Read block 324 len 28672 05/30 Write block 187 len 16384 data: 7 6 7 6 06/30 Read block 366 len 40960 07/30 Read block 501 len 28672 08/30 Write block 397 len 20480 data: 12 14 12 13 12 09/30 Read block 252 len 12288 10/30 Read block 429 len 24576 11/30 Read block 539 len 24576 12/30 Read block 51 len 4096 13/30 Read block 536 len 28672 14/30 Read block 466 len 32768 15/30 Write block 138 len 36864 data: 10 12 11 13 12 15 13 13 13 16/30 Write block 9 len 8192 data: 9 10 17/30 Read block 196 len 32768 18/30 Write block 84 len 36864 data: 9 7 9 10 10 13 11 11 9 19/30 Read block 572 len 24576 20/30 Read block 66 len 36864 21/30 Flush 22/30 Write block 249 len 8192 data: 13 12 23/30 Write block 330 len 36864 data: 6 9 9 11 8 8 9 11 13 24/30 Write block 385 len 12288 data: 5 5 5 25/30 Read block 110 len 4096 26/30 Read block 273 len 28672 27/30 Write block 216 len 40960 data: 6 6 6 5 3 5 6 7 11 10 28/30 Write block 430 len 16384 data: 9 11 7 6 29/30 Read block 312 len 12288 30/30 Write block 346 len 12288 data: 14 11 12 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:05:57.61285235Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 199 199 199 F F F 2 040-059 106 106 106 197 197 197 F F F 3 060-079 106 106 106 197 197 197 F F F 4 080-099 108 108 108 200 200 200 F F F 5 100-119 108 108 108 200 200 200 F F F 6 120-139 108 108 108 200 200 200 F F F 7 140-159 108 108 108 200 200 200 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 108 108 108 200 200 200 F F F 10 200-219 96 96 96 183 183 183 T T T 11 220-239 98 98 98 186 186 186 T T T 12 240-259 102 102 102 192 192 192 T T T 13 260-279 107 107 107 199 199 199 F F F 14 280-299 106 106 106 197 197 197 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 106 106 106 196 196 196 T T T 17 340-359 105 105 105 195 195 195 T T T 18 360-379 108 108 108 200 200 200 F F F 19 380-399 108 108 108 200 200 200 T T T 20 400-419 108 108 108 200 200 200 F F F 21 420-439 98 98 98 186 186 186 T T T 22 440-459 106 106 106 197 197 197 F F F 23 460-479 105 105 105 195 195 195 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 105 105 105 195 195 195 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 108, Max flush: 200 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 199 199 199 F F F 2 040-059 106 106 106 197 197 197 F F F 3 060-079 106 106 106 197 197 197 F F F 4 080-099 108 108 108 200 200 200 F F F 5 100-119 108 108 108 200 200 200 F F F 6 120-139 108 108 108 200 200 200 F F F 7 140-159 108 108 108 200 200 200 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 108 108 108 200 200 200 F F F 10 200-219 109 109 109 201 201 201 F F F 11 220-239 109 109 109 201 201 201 F F F 12 240-259 109 109 109 201 201 201 F F F 13 260-279 107 107 107 199 199 199 F F F 14 280-299 106 106 106 197 197 197 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 109 109 109 201 201 201 F F F 17 340-359 109 109 109 201 201 201 F F F 18 360-379 108 108 108 200 200 200 F F F 19 380-399 109 109 109 201 201 201 F F F 20 400-419 108 108 108 200 200 200 F F F 21 420-439 109 109 109 201 201 201 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 105 105 105 195 195 195 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 105 105 105 195 195 195 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 109, Max flush: 201 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.86012874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.903149546Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.9037879Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.903829739Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.903843918Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.903858378Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} {"msg":"Crucible 9a37c6d4-f216-4126-87bb-0183d44911bb has session id: 2b1c46f7-94b3-4f17-a596-c9d596bcde7e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.904333455Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Upstairs opts: Upstairs UUID: 9a37c6d4-f216-4126-87bb-0183d44911bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.904398573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Crucible stats registered with UUID: 9a37c6d4-f216-4126-87bb-0183d44911bb","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.904418083Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:11.904430663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.904892091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.90491594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} {"msg":"9a37c6d4-f216-4126-87bb-0183d44911bb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.904992388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905022377Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"0","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905044467Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"0","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905058126Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"1","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905071916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"1","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905084645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905128074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905148804Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905174143Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905200532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44130)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905234642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54795)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905264911Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:11.90529327Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:11.905322379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:11.905434766Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.905596462Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.906009021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"1","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.9060375Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"0","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.906609255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"1","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.906752782Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"0","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.907118652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:11.907363625Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.906579949Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55446)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.906640817Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.907637471Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.908305953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[0]R flush_numbers[0..12]: [200, 199, 197, 197, 200, 200, 200, 200, 175, 200, 201, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.908928347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[0]R generation[0..12]: [108, 107, 106, 106, 108, 108, 108, 108, 92, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.908957496Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.908974076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[1]R flush_numbers[0..12]: [200, 199, 197, 197, 200, 200, 200, 200, 175, 200, 201, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.908988395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[1]R generation[0..12]: [108, 107, 106, 106, 108, 108, 108, 108, 92, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909002645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909016524Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[2]R flush_numbers[0..12]: [200, 199, 197, 197, 200, 200, 200, 200, 175, 200, 201, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909030174Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[2]R generation[0..12]: [108, 107, 106, 106, 108, 108, 108, 108, 92, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909043654Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909057334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909069953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909083023Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909095922Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909112832Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"":"downstairs","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909126542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909138221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"0","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909151591Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"1","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909164451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"client":"2","":"downstairs","session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.90917746Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.90918955Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"9a37c6d4-f216-4126-87bb-0183d44911bb is now active with session: 2b1c46f7-94b3-4f17-a596-c9d596bcde7e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.90920207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909213659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632,"session_id":"2b1c46f7-94b3-4f17-a596-c9d596bcde7e"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:06:21.909225449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} {"msg":"Activated sub_volume 9a37c6d4-f216-4126-87bb-0183d44911bb","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:21.909237989Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 270 len 36864 data: 14 10 9 11 12 12 10 10 8 02/30 Write block 152 len 24576 data: 6 5 5 6 6 8 03/30 Read block 316 len 24576 04/30 Read block 478 len 24576 05/30 Write block 125 len 40960 data: 8 5 9 9 8 6 7 9 9 8 06/30 Write block 363 len 4096 data: 12 07/30 Read block 454 len 24576 08/30 Read block 230 len 24576 09/30 Write block 191 len 4096 data: 7 10/30 Write block 62 len 12288 data: 11 11 11 11/30 Read block 331 len 12288 12/30 Read block 436 len 20480 13/30 Read block 377 len 16384 14/30 Flush 15/30 Write block 410 len 36864 data: 11 10 9 10 7 8 10 10 11 16/30 Write block 475 len 20480 data: 10 8 9 9 11 17/30 Read block 470 len 20480 18/30 Read block 46 len 8192 19/30 Read block 58 len 16384 20/30 Flush 21/30 Flush 22/30 Write block 380 len 16384 data: 13 11 9 8 23/30 Read block 262 len 40960 24/30 Write block 246 len 32768 data: 10 14 14 14 13 11 11 8 25/30 Read block 168 len 24576 26/30 Flush 27/30 Write block 78 len 32768 data: 10 9 8 8 10 11 10 8 28/30 Read block 289 len 24576 29/30 Read block 224 len 40960 30/30 Read block 15 len 36864 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:21.932046358Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 199 199 199 F F F 2 040-059 106 106 106 197 197 197 F F F 3 060-079 110 110 110 202 202 202 T T T 4 080-099 108 108 108 200 200 200 T T T 5 100-119 108 108 108 200 200 200 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 110 110 110 202 202 202 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 109 109 109 201 201 201 F F F 11 220-239 109 109 109 201 201 201 F F F 12 240-259 110 110 110 205 205 205 F F F 13 260-279 110 110 110 202 202 202 F F F 14 280-299 106 106 106 197 197 197 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 109 109 109 201 201 201 F F F 17 340-359 109 109 109 201 201 201 F F F 18 360-379 110 110 110 202 202 202 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 110 110 110 203 203 203 F F F 21 420-439 109 109 109 201 201 201 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 110 110 110 203 203 203 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 105 105 105 195 195 195 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 110, Max flush: 205 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 199 199 199 F F F 2 040-059 106 106 106 197 197 197 F F F 3 060-079 111 111 111 206 206 206 F F F 4 080-099 111 111 111 206 206 206 F F F 5 100-119 108 108 108 200 200 200 F F F 6 120-139 110 110 110 202 202 202 F F F 7 140-159 110 110 110 202 202 202 F F F 8 160-179 92 92 92 175 175 175 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 109 109 109 201 201 201 F F F 11 220-239 109 109 109 201 201 201 F F F 12 240-259 110 110 110 205 205 205 F F F 13 260-279 110 110 110 202 202 202 F F F 14 280-299 106 106 106 197 197 197 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 109 109 109 201 201 201 F F F 17 340-359 109 109 109 201 201 201 F F F 18 360-379 110 110 110 202 202 202 F F F 19 380-399 110 110 110 205 205 205 F F F 20 400-419 110 110 110 203 203 203 F F F 21 420-439 109 109 109 201 201 201 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 110 110 110 203 203 203 F F F 24 480-499 101 101 101 190 190 190 F F F 25 500-519 102 102 102 192 192 192 F F F 26 520-539 105 105 105 195 195 195 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 111, Max flush: 206 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.158017006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:36.200625404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.201075512Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.232327988Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.233090198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.233123248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.233136647Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.233151037Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} {"msg":"Crucible cbe11d4d-b6ad-4da9-a902-2c9658800dca has session id: 8ad3647b-0100-4393-b7d5-00c3ea6fd438","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.233839678Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Upstairs opts: Upstairs UUID: cbe11d4d-b6ad-4da9-a902-2c9658800dca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.233871688Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Crucible stats registered with UUID: cbe11d4d-b6ad-4da9-a902-2c9658800dca","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.233887037Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:36.233902607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234441273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234464952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} {"msg":"cbe11d4d-b6ad-4da9-a902-2c9658800dca active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.23453199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.23454885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"0","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234571979Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"0","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234594759Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"1","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234621658Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"1","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234646357Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"2","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234674776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"2","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234703276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234731735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234758724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"ds_connection connected from Ok(127.0.0.1:59798)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234786084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"ds_connection connected from Ok(127.0.0.1:65004)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234812873Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"ds_connection connected from Ok(127.0.0.1:53715)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.234884681Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.2356898Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"1","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.235729979Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"2","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.235746968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"0","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.236472059Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"1","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.236505478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"0","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.236536287Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"2","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237112712Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.23720421Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[0]R flush_numbers[0..12]: [200, 199, 197, 206, 206, 200, 202, 202, 175, 202, 201, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237290478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[0]R generation[0..12]: [108, 107, 106, 111, 111, 108, 110, 110, 92, 110, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237310207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237324657Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[1]R flush_numbers[0..12]: [200, 199, 197, 206, 206, 200, 202, 202, 175, 202, 201, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237338177Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[1]R generation[0..12]: [108, 107, 106, 111, 111, 108, 110, 110, 92, 110, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237351796Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237370225Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[2]R flush_numbers[0..12]: [200, 199, 197, 206, 206, 200, 202, 202, 175, 202, 201, 201]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237384295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[2]R generation[0..12]: [108, 107, 106, 111, 111, 108, 110, 110, 92, 110, 109, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237397805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237411564Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237424194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237437264Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237449903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237462473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"":"downstairs","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237475723Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237487433Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"0","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237500572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"client":"1","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237513682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":Wait for a query_work_queue command to finish before sending IO 1640,"client":"2","":"downstairs","session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237534731Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237546701Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"cbe11d4d-b6ad-4da9-a902-2c9658800dca is now active with session: 8ad3647b-0100-4393-b7d5-00c3ea6fd438","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.237559191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.23757275Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640,"session_id":"8ad3647b-0100-4393-b7d5-00c3ea6fd438"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.23758514Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} {"msg":"Activated sub_volume cbe11d4d-b6ad-4da9-a902-2c9658800dca","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:36.23759611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 436 len 8192 data: 8 8 02/30 Read block 232 len 24576 03/30 Write block 294 len 4096 data: 8 04/30 Write block 171 len 20480 data: 11 9 10 9 10 05/30 Write block 123 len 40960 data: 7 7 9 6 10 10 9 7 8 10 06/30 Read block 517 len 20480 07/30 Flush 08/30 Write block 63 len 32768 data: 12 12 11 12 11 11 11 7 09/30 Write block 53 len 4096 data: 10 10/30 Write block 517 len 40960 data: 8 9 9 7 7 7 8 8 8 6 11/30 Write block 491 len 20480 data: 5 4 7 10 12 12/30 Read block 559 len 40960 13/30 Read block 305 len 4096 14/30 Read block 415 len 28672 15/30 Write block 133 len 12288 data: 10 9 8 16/30 Write block 335 len 28672 data: 9 10 12 14 10 13 13 17/30 Read block 521 len 24576 18/30 Read block 504 len 40960 19/30 Flush 20/30 Write block 322 len 40960 data: 7 8 8 7 9 6 7 7 7 10 21/30 Write block 87 len 36864 data: 11 11 14 12 12 10 12 10 10 22/30 Write block 166 len 4096 data: 8 23/30 Read block 315 len 8192 24/30 Write block 217 len 28672 data: 7 7 6 4 6 7 8 25/30 Write block 73 len 16384 data: 6 9 12 10 26/30 Flush 27/30 Write block 294 len 20480 data: 9 7 7 5 5 28/30 Read block 122 len 16384 29/30 Write block 378 len 16384 data: 13 14 14 12 30/30 Write block 390 len 36864 data: 5 9 10 10 14 14 11 13 15 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Sent Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 2 2 3 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:36.257492226Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 199 199 199 F F F 2 040-059 112 112 112 208 208 208 F F F 3 060-079 112 112 112 209 209 209 F F F 4 080-099 112 112 112 209 209 209 F F F 5 100-119 108 108 108 200 200 200 F F F 6 120-139 112 112 112 208 208 208 F F F 7 140-159 110 110 110 202 202 202 F F F 8 160-179 112 112 112 209 209 209 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 112 112 112 209 209 209 F F F 12 240-259 110 110 110 205 205 205 F F F 13 260-279 110 110 110 202 202 202 F F F 14 280-299 112 112 112 207 207 207 T T T 15 300-319 107 107 107 199 199 199 F F F 16 320-339 112 112 112 209 209 209 F F F 17 340-359 112 112 112 208 208 208 F F F 18 360-379 110 110 110 202 202 202 T T T 19 380-399 110 110 110 205 205 205 T T T 20 400-419 110 110 110 203 203 203 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 110 110 110 203 203 203 F F F 24 480-499 112 112 112 208 208 208 F F F 25 500-519 112 112 112 208 208 208 F F F 26 520-539 112 112 112 208 208 208 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 112, Max flush: 209 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 107 107 107 199 199 199 F F F 2 040-059 112 112 112 208 208 208 F F F 3 060-079 112 112 112 209 209 209 F F F 4 080-099 112 112 112 209 209 209 F F F 5 100-119 108 108 108 200 200 200 F F F 6 120-139 112 112 112 208 208 208 F F F 7 140-159 110 110 110 202 202 202 F F F 8 160-179 112 112 112 209 209 209 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 112 112 112 209 209 209 F F F 11 220-239 112 112 112 209 209 209 F F F 12 240-259 110 110 110 205 205 205 F F F 13 260-279 110 110 110 202 202 202 F F F 14 280-299 113 113 113 210 210 210 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 112 112 112 209 209 209 F F F 17 340-359 112 112 112 208 208 208 F F F 18 360-379 113 113 113 210 210 210 F F F 19 380-399 113 113 113 210 210 210 F F F 20 400-419 110 110 110 203 203 203 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 110 110 110 203 203 203 F F F 24 480-499 112 112 112 208 208 208 F F F 25 500-519 112 112 112 208 208 208 F F F 26 520-539 112 112 112 208 208 208 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 113, Max flush: 210 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.515307704Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:40.558041819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.558457048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.589782793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.590361207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.590400946Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.590415626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.590430016Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} {"msg":"Crucible 53ed75a6-45ee-42cc-a277-abdee49728a2 has session id: ce4ec762-32ce-4905-b3eb-97359690fc36","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591052109Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Upstairs opts: Upstairs UUID: 53ed75a6-45ee-42cc-a277-abdee49728a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591086528Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Crucible stats registered with UUID: 53ed75a6-45ee-42cc-a277-abdee49728a2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591102338Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:40.591114807Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591504467Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591526607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} {"msg":"53ed75a6-45ee-42cc-a277-abdee49728a2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591593505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591610045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"0","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591632444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"0","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591645844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"1","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591660283Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"1","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591684092Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"2","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591699792Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"2","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591721891Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591747971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.59177531Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"ds_connection connected from Ok(127.0.0.1:55003)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591802009Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"ds_connection connected from Ok(127.0.0.1:34066)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.591825839Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"ds_connection connected from Ok(127.0.0.1:55244)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.592064022Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.59254964Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"1","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.592577669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"2","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.592796573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"0","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.5933075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"1","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.593341029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"0","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.593356148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"2","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.593912464Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594037061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[0]R flush_numbers[0..12]: [200, 199, 208, 209, 209, 200, 208, 202, 209, 202, 209, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594155447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[0]R generation[0..12]: [108, 107, 112, 112, 112, 108, 112, 110, 112, 110, 112, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594174607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594189006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[1]R flush_numbers[0..12]: [200, 199, 208, 209, 209, 200, 208, 202, 209, 202, 209, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594202626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[1]R generation[0..12]: [108, 107, 112, 112, 112, 108, 112, 110, 112, 110, 112, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594215976Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594229536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[2]R flush_numbers[0..12]: [200, 199, 208, 209, 209, 200, 208, 202, 209, 202, 209, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594242915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[2]R generation[0..12]: [108, 107, 112, 112, 112, 108, 112, 110, 112, 110, 112, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594256135Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594269654Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594282314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594295244Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594308003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594330383Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"":"downstairs","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594343853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594355622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"0","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594368422Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"1","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594381201Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"client":"2","":"downstairs","session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594394101Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594405891Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"53ed75a6-45ee-42cc-a277-abdee49728a2 is now active with session: ce4ec762-32ce-4905-b3eb-97359690fc36","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594418231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.59442975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648,"session_id":"ce4ec762-32ce-4905-b3eb-97359690fc36"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.59444201Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} {"msg":"Activated sub_volume 53ed75a6-45ee-42cc-a277-abdee49728a2","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:40.594453099Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 32768 data: 7 8 9 7 7 7 7 8 02/30 Read block 368 len 20480 03/30 Write block 36 len 4096 data: 11 04/30 Flush 05/30 Write block 46 len 8192 data: 6 7 06/30 Read block 14 len 40960 07/30 Read block 386 len 12288 08/30 Write block 205 len 4096 data: 7 09/30 Flush 10/30 Write block 98 len 28672 data: 12 12 13 13 12 14 15 11/30 Flush 12/30 Flush 13/30 Write block 527 len 12288 data: 8 9 9 14/30 Write block 411 len 4096 data: 11 15/30 Read block 62 len 20480 16/30 Read block 295 len 40960 17/30 Write block 55 len 24576 data: 9 7 8 6 8 8 18/30 Read block 259 len 32768 19/30 Write block 96 len 32768 data: 13 13 13 13 14 14 13 15 20/30 Read block 491 len 4096 21/30 Read block 342 len 32768 22/30 Read block 527 len 16384 23/30 Read block 211 len 32768 24/30 Flush 25/30 Write block 263 len 40960 data: 12 11 10 12 13 13 13 15 11 10 26/30 Read block 303 len 4096 27/30 Write block 101 len 8192 data: 15 14 28/30 Read block 2 len 4096 29/30 Read block 9 len 20480 30/30 Read block 213 len 12288 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:40.613950196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 114 114 114 215 215 215 F F F 3 060-079 114 114 114 215 215 215 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 114 114 114 215 215 215 T T T 6 120-139 112 112 112 208 208 208 F F F 7 140-159 110 110 110 202 202 202 F F F 8 160-179 112 112 112 209 209 209 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 114 114 114 212 212 212 F F F 11 220-239 112 112 112 209 209 209 F F F 12 240-259 110 110 110 205 205 205 F F F 13 260-279 110 110 110 202 202 202 T T T 14 280-299 113 113 113 210 210 210 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 112 112 112 209 209 209 F F F 17 340-359 112 112 112 208 208 208 F F F 18 360-379 113 113 113 210 210 210 F F F 19 380-399 113 113 113 210 210 210 F F F 20 400-419 114 114 114 215 215 215 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 110 110 110 203 203 203 F F F 24 480-499 112 112 112 208 208 208 F F F 25 500-519 112 112 112 208 208 208 F F F 26 520-539 114 114 114 215 215 215 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 114, Max flush: 215 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 114 114 114 215 215 215 F F F 3 060-079 114 114 114 215 215 215 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 115 115 115 216 216 216 F F F 6 120-139 112 112 112 208 208 208 F F F 7 140-159 110 110 110 202 202 202 F F F 8 160-179 112 112 112 209 209 209 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 114 114 114 212 212 212 F F F 11 220-239 112 112 112 209 209 209 F F F 12 240-259 110 110 110 205 205 205 F F F 13 260-279 115 115 115 216 216 216 F F F 14 280-299 113 113 113 210 210 210 F F F 15 300-319 107 107 107 199 199 199 F F F 16 320-339 112 112 112 209 209 209 F F F 17 340-359 112 112 112 208 208 208 F F F 18 360-379 113 113 113 210 210 210 F F F 19 380-399 113 113 113 210 210 210 F F F 20 400-419 114 114 114 215 215 215 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 110 110 110 203 203 203 F F F 24 480-499 112 112 112 208 208 208 F F F 25 500-519 112 112 112 208 208 208 F F F 26 520-539 114 114 114 215 215 215 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 101 101 101 190 190 190 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 115, Max flush: 216 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.864176275Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.907685319Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.90838782Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.908421909Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.908435769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.908450168Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} {"msg":"Crucible 31ed536b-f3e1-45ee-a215-cc090ff7d522 has session id: 4d643d45-7ced-4488-9920-6aabdc502244","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.908947495Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Upstairs opts: Upstairs UUID: 31ed536b-f3e1-45ee-a215-cc090ff7d522, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.908988204Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Crucible stats registered with UUID: 31ed536b-f3e1-45ee-a215-cc090ff7d522","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909005534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:44.909018813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909417943Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909444912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} {"msg":"31ed536b-f3e1-45ee-a215-cc090ff7d522 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.90951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.90953314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"0","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909553409Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"0","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909572749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"1","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909598518Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"1","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909624088Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909695876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909721325Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909743884Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909765374Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:44.909791653Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"ds_connection connected from Ok(127.0.0.1:52834)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909816952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:44.909844542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"ds_connection connected from Ok(127.0.0.1:60575)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909869131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:44.909938029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.909975608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.910424996Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"1","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.910466085Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"0","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.911165447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"1","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.911192706Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"0","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.911775131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:44.91181255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.911692395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"ds_connection connected from Ok(127.0.0.1:47702)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.911815402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.912773166Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.913428139Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[0]R flush_numbers[0..12]: [200, 211, 215, 215, 215, 216, 208, 202, 209, 202, 212, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914075762Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[0]R generation[0..12]: [108, 114, 114, 114, 114, 115, 112, 110, 112, 110, 114, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914097992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914117781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[1]R flush_numbers[0..12]: [200, 211, 215, 215, 215, 216, 208, 202, 209, 202, 212, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914132811Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[1]R generation[0..12]: [108, 114, 114, 114, 114, 115, 112, 110, 112, 110, 114, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.91414677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914161Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[2]R flush_numbers[0..12]: [200, 211, 215, 215, 215, 216, 208, 202, 209, 202, 212, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.91417461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[2]R generation[0..12]: [108, 114, 114, 114, 114, 115, 112, 110, 112, 110, 114, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914188319Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914202229Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914215068Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914228048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914240718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914253558Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"":"downstairs","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914266577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914278427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"0","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914291606Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"1","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914308426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"client":"2","":"downstairs","session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914330076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914343435Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"31ed536b-f3e1-45ee-a215-cc090ff7d522 is now active with session: 4d643d45-7ced-4488-9920-6aabdc502244","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914355985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914367895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656,"session_id":"4d643d45-7ced-4488-9920-6aabdc502244"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914379894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} {"msg":"Activated sub_volume 31ed536b-f3e1-45ee-a215-cc090ff7d522","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:54.914390854Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 560 len 20480 data: 3 4 7 9 11 02/30 Flush 03/30 Write block 142 len 24576 data: 13 16 14 14 14 13 04/30 Write block 533 len 28672 data: 10 10 12 13 12 12 13 05/30 Write block 339 len 28672 data: 11 14 14 14 12 15 14 06/30 Flush 07/30 Write block 145 len 16384 data: 15 15 14 9 08/30 Write block 323 len 12288 data: 9 9 8 09/30 Read block 193 len 20480 10/30 Write block 73 len 24576 data: 7 10 13 11 10 11 11/30 Flush 12/30 Write block 466 len 28672 data: 5 5 3 4 6 5 5 13/30 Read block 293 len 32768 14/30 Read block 279 len 28672 15/30 Read block 520 len 24576 16/30 Read block 450 len 24576 17/30 Read block 95 len 12288 18/30 Read block 232 len 8192 19/30 Read block 165 len 8192 20/30 Write block 156 len 32768 data: 7 9 8 7 5 3 6 5 21/30 Write block 339 len 28672 data: 12 15 15 15 13 16 15 22/30 Flush 23/30 Flush 24/30 Write block 481 len 36864 data: 10 8 7 7 7 9 7 6 5 25/30 Write block 238 len 36864 data: 8 6 7 8 8 10 6 9 11 26/30 Write block 309 len 8192 data: 14 12 27/30 Write block 304 len 16384 data: 10 11 11 11 28/30 Read block 416 len 20480 29/30 Write block 219 len 24576 data: 7 5 7 8 9 12 30/30 Read block 226 len 16384 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 5 7 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:54.935637274Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 114 114 114 215 215 215 F F F 3 060-079 116 116 116 219 219 219 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 115 115 115 216 216 216 F F F 6 120-139 112 112 112 208 208 208 F F F 7 140-159 116 116 116 220 220 220 F F F 8 160-179 116 116 116 220 220 220 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 114 114 114 212 212 212 T T T 11 220-239 112 112 112 209 209 209 T T T 12 240-259 110 110 110 205 205 205 T T T 13 260-279 115 115 115 216 216 216 F F F 14 280-299 113 113 113 210 210 210 F F F 15 300-319 107 107 107 199 199 199 T T T 16 320-339 116 116 116 220 220 220 F F F 17 340-359 116 116 116 220 220 220 F F F 18 360-379 113 113 113 210 210 210 F F F 19 380-399 113 113 113 210 210 210 F F F 20 400-419 114 114 114 215 215 215 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 116 116 116 220 220 220 F F F 24 480-499 112 112 112 208 208 208 T T T 25 500-519 112 112 112 208 208 208 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 116 116 116 217 217 217 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 116, Max flush: 220 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 200 200 200 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 114 114 114 215 215 215 F F F 3 060-079 116 116 116 219 219 219 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 115 115 115 216 216 216 F F F 6 120-139 112 112 112 208 208 208 F F F 7 140-159 116 116 116 220 220 220 F F F 8 160-179 116 116 116 220 220 220 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 117 117 117 221 221 221 F F F 11 220-239 117 117 117 221 221 221 F F F 12 240-259 117 117 117 221 221 221 F F F 13 260-279 115 115 115 216 216 216 F F F 14 280-299 113 113 113 210 210 210 F F F 15 300-319 117 117 117 221 221 221 F F F 16 320-339 116 116 116 220 220 220 F F F 17 340-359 116 116 116 220 220 220 F F F 18 360-379 113 113 113 210 210 210 F F F 19 380-399 113 113 113 210 210 210 F F F 20 400-419 114 114 114 215 215 215 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 116 116 116 220 220 220 F F F 24 480-499 117 117 117 221 221 221 F F F 25 500-519 112 112 112 208 208 208 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 116 116 116 217 217 217 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 117, Max flush: 221 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.173658715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:59.215577831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.215952801Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.24709411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.247815451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.24784984Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.24786383Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.24787805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} {"msg":"Crucible 6f1cccd2-c9e1-45ab-bdcc-cb60f9d12831 has session id: 272f515b-f7cc-4737-aa92-afdc99724799","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.248474344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Upstairs opts: Upstairs UUID: 6f1cccd2-c9e1-45ab-bdcc-cb60f9d12831, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.248510083Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Crucible stats registered with UUID: 6f1cccd2-c9e1-45ab-bdcc-cb60f9d12831","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.248526433Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:59.248539282Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.248940652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.248968491Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} {"msg":"6f1cccd2-c9e1-45ab-bdcc-cb60f9d12831 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249046179Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249067308Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"0","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249095078Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"0","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249122637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"1","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249154456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"1","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249235704Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"2","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249270623Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"2","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249296352Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249325672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249355771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"ds_connection connected from Ok(127.0.0.1:51081)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.24938674Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"ds_connection connected from Ok(127.0.0.1:39877)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249417039Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"ds_connection connected from Ok(127.0.0.1:47588)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.249446928Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.250045053Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"1","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.250186439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"2","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.250405133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"0","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.250737494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"1","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.250776883Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"2","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251058226Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"0","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251346368Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251445526Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[0]R flush_numbers[0..12]: [200, 211, 215, 219, 215, 216, 208, 220, 220, 202, 221, 221]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251604492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[0]R generation[0..12]: [108, 114, 114, 116, 114, 115, 112, 116, 116, 110, 117, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251622781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251636861Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[1]R flush_numbers[0..12]: [200, 211, 215, 219, 215, 216, 208, 220, 220, 202, 221, 221]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.25165046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[1]R generation[0..12]: [108, 114, 114, 116, 114, 115, 112, 116, 116, 110, 117, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.25166388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.25167745Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[2]R flush_numbers[0..12]: [200, 211, 215, 219, 215, 216, 208, 220, 220, 202, 221, 221]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251690729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[2]R generation[0..12]: [108, 114, 114, 116, 114, 115, 112, 116, 116, 110, 117, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251704079Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251721509Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251734638Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251747718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251760297Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251773047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"":"downstairs","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251785967Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251797616Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"0","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251810346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"1","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251822926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"client":"2","":"downstairs","session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {Wait for a query_work_queue command to finish before sending IO "msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251842255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251854315Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"6f1cccd2-c9e1-45ab-bdcc-cb60f9d12831 is now active with session: 272f515b-f7cc-4737-aa92-afdc99724799","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251866735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251878374Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664,"session_id":"272f515b-f7cc-4737-aa92-afdc99724799"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251890404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} {"msg":"Activated sub_volume 6f1cccd2-c9e1-45ab-bdcc-cb60f9d12831","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:06:59.251901404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 332 len 32768 data: 10 12 9 10 11 13 15 13 02/30 Read block 299 len 24576 03/30 Write block 224 len 24576 data: 13 11 12 12 9 7 04/30 Write block 76 len 16384 data: 12 11 12 10 05/30 Write block 363 len 20480 data: 13 9 8 9 11 06/30 Flush 07/30 Write block 493 len 28672 data: 8 11 13 12 10 8 4 08/30 Write block 12 len 24576 data: 8 8 9 11 10 10 09/30 Write block 407 len 28672 data: 11 12 9 12 12 10 11 10/30 Write block 475 len 12288 data: 11 9 10 11/30 Write block 185 len 4096 data: 8 12/30 Write block 505 len 32768 data: 7 6 7 11 10 10 13 10 13/30 Read block 95 len 40960 14/30 Read block 396 len 4096 15/30 Flush 16/30 Write block 302 len 16384 data: 7 8 11 12 17/30 Write block 286 len 4096 data: 4 18/30 Read block 390 len 32768 19/30 Write block 347 len 16384 data: 12 13 11 9 20/30 Read block 270 len 16384 21/30 Write block 561 len 32768 data: 5 8 10 12 11 11 10 11 22/30 Read block 50 len 20480 23/30 Read block 593 len 28672 24/30 Write block 142 len 4096 data: 14 25/30 Write block 386 len 24576 data: 6 6 4 4 6 10 26/30 Read block 343 len 32768 27/30 Read block 192 len 4096 28/30 Read block 574 len 8192 29/30 Read block 511 len 12288 30/30 Read block 319 len 32768 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 4 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:06:59.271731071Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 223 223 223 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 114 114 114 215 215 215 F F F 3 060-079 118 118 118 222 222 222 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 115 115 115 216 216 216 F F F 6 120-139 112 112 112 208 208 208 F F F 7 140-159 116 116 116 220 220 220 T T T 8 160-179 116 116 116 220 220 220 F F F 9 180-199 118 118 118 223 223 223 F F F 10 200-219 117 117 117 221 221 221 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 117 117 117 221 221 221 F F F 13 260-279 115 115 115 216 216 216 F F F 14 280-299 113 113 113 210 210 210 T T T 15 300-319 117 117 117 221 221 221 T T T 16 320-339 118 118 118 222 222 222 F F F 17 340-359 116 116 116 220 220 220 T T T 18 360-379 118 118 118 222 222 222 F F F 19 380-399 113 113 113 210 210 210 T T T 20 400-419 118 118 118 223 223 223 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 118 118 118 223 223 223 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 116 116 116 217 217 217 T T T 29 580-599 106 106 106 198 198 198 F F F Max gen: 118, Max flush: 223 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 223 223 223 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 114 114 114 215 215 215 F F F 3 060-079 118 118 118 222 222 222 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 115 115 115 216 216 216 F F F 6 120-139 112 112 112 208 208 208 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 116 116 116 220 220 220 F F F 9 180-199 118 118 118 223 223 223 F F F 10 200-219 117 117 117 221 221 221 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 117 117 117 221 221 221 F F F 13 260-279 115 115 115 216 216 216 F F F 14 280-299 119 119 119 224 224 224 F F F 15 300-319 119 119 119 224 224 224 F F F 16 320-339 118 118 118 222 222 222 F F F 17 340-359 119 119 119 224 224 224 F F F 18 360-379 118 118 118 222 222 222 F F F 19 380-399 119 119 119 224 224 224 F F F 20 400-419 118 118 118 223 223 223 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 118 118 118 223 223 223 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 119 119 119 224 224 224 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 119, Max flush: 224 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.549141284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:03.591187206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.591579526Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.622370075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.622957589Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.622992358Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.623009808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.623024588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} {"msg":"Crucible 0691e997-19a0-4e67-92f7-19dd310af205 has session id: cf4730c5-971b-4ef5-89db-35603a4eb666","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.623657771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Upstairs opts: Upstairs UUID: 0691e997-19a0-4e67-92f7-19dd310af205, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.62370241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Crucible stats registered with UUID: 0691e997-19a0-4e67-92f7-19dd310af205","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.623720929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:03.623733649Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.62408994Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624113959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} {"msg":"0691e997-19a0-4e67-92f7-19dd310af205 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624163908Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624190937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"0","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624215426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"0","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624230476Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"1","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624244536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"1","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624257405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"2","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624274065Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"2","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624287554Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624301434Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624317754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"ds_connection connected from Ok(127.0.0.1:63570)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624434631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"ds_connection connected from Ok(127.0.0.1:62747)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.62447499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"ds_connection connected from Ok(127.0.0.1:56120)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.624505819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.625186461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"1","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.62521403Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"2","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.625229199Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"0","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.625844253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"1","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.62598564Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"0","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626115676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"2","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626434388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626557024Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[0]R flush_numbers[0..12]: [223, 211, 215, 222, 215, 216, 208, 224, 220, 223, 221, 222]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626677521Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[0]R generation[0..12]: [118, 114, 114, 118, 114, 115, 112, 119, 116, 118, 117, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626698521Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626713431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[1]R flush_numbers[0..12]: [223, 211, 215, 222, 215, 216, 208, 224, 220, 223, 221, 222]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.62672729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[1]R generation[0..12]: [118, 114, 114, 118, 114, 115, 112, 119, 116, 118, 117, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.62674105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626754779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[2]R flush_numbers[0..12]: [223, 211, 215, 222, 215, 216, 208, 224, 220, 223, 221, 222]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626768569Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[2]R generation[0..12]: [118, 114, 114, 118, 114, 115, 112, 119, 116, 118, 117, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626782879Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626796738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626809738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626823108Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626835907Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:07:03.626848557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"":"downstairs","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626882386Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626897336Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"0","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626911015Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"1","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626923945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"client":"2","":"downstairs","session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626936914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626948844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"0691e997-19a0-4e67-92f7-19dd310af205 is now active with session: cf4730c5-971b-4ef5-89db-35603a4eb666","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626961624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626973304Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672,"session_id":"cf4730c5-971b-4ef5-89db-35603a4eb666"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626985553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} {"msg":"Activated sub_volume 0691e997-19a0-4e67-92f7-19dd310af205","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:03.626996673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 295 len 40960 data: 8 8 6 6 5 6 5 8 9 12 02/30 Write block 385 len 16384 data: 6 7 7 5 03/30 Write block 250 len 28672 data: 14 12 12 9 8 7 8 04/30 Write block 301 len 32768 data: 6 9 10 13 13 12 12 12 05/30 Write block 275 len 8192 data: 13 11 06/30 Write block 63 len 32768 data: 13 13 12 13 12 12 12 8 07/30 Write block 43 len 20480 data: 10 10 10 7 8 08/30 Read block 65 len 12288 09/30 Read block 587 len 36864 10/30 Read block 221 len 32768 11/30 Write block 293 len 36864 data: 8 10 9 9 7 7 6 7 7 12/30 Read block 160 len 16384 13/30 Write block 54 len 28672 data: 10 10 8 9 7 9 9 14/30 Read block 299 len 8192 15/30 Read block 366 len 4096 16/30 Read block 506 len 40960 17/30 Write block 121 len 4096 data: 9 18/30 Write block 461 len 20480 data: 7 7 8 6 4 19/30 Read block 494 len 40960 20/30 Read block 432 len 24576 21/30 Flush 22/30 Write block 258 len 24576 data: 7 10 10 14 15 13 23/30 Write block 272 len 40960 data: 11 12 13 14 12 11 9 8 7 6 24/30 Read block 434 len 36864 25/30 Flush 26/30 Write block 173 len 32768 data: 11 10 11 9 7 10 10 9 27/30 Write block 398 len 36864 data: 16 13 14 13 10 11 10 10 9 28/30 Read block 561 len 16384 29/30 Write block 366 len 24576 data: 10 12 11 8 10 10 30/30 Read block 294 len 36864 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 4 3 5 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:03.648127186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 223 223 223 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 115 115 115 216 216 216 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 116 116 116 220 220 220 T T T 9 180-199 118 118 118 223 223 223 T T T 10 200-219 117 117 117 221 221 221 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 120 120 120 226 226 226 F F F 13 260-279 120 120 120 226 226 226 F F F 14 280-299 120 120 120 226 226 226 F F F 15 300-319 120 120 120 225 225 225 F F F 16 320-339 118 118 118 222 222 222 F F F 17 340-359 119 119 119 224 224 224 F F F 18 360-379 118 118 118 222 222 222 T T T 19 380-399 120 120 120 225 225 225 T T T 20 400-419 118 118 118 223 223 223 T T T 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 120 120 120 225 225 225 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 119 119 119 224 224 224 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 120, Max flush: 226 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 223 223 223 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 114 114 114 215 215 215 F F F 5 100-119 115 115 115 216 216 216 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 121 121 121 227 227 227 F F F 9 180-199 121 121 121 227 227 227 F F F 10 200-219 117 117 117 221 221 221 F F F 11 220-239 118 118 118 222 222 222 F F F 12 240-259 120 120 120 226 226 226 F F F 13 260-279 120 120 120 226 226 226 F F F 14 280-299 120 120 120 226 226 226 F F F 15 300-319 120 120 120 225 225 225 F F F 16 320-339 118 118 118 222 222 222 F F F 17 340-359 119 119 119 224 224 224 F F F 18 360-379 121 121 121 227 227 227 F F F 19 380-399 121 121 121 227 227 227 F F F 20 400-419 121 121 121 227 227 227 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 106 106 106 197 197 197 F F F 23 460-479 120 120 120 225 225 225 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 119 119 119 224 224 224 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 121, Max flush: 227 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.914743714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:07.95663313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.957040899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.987802329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.988378884Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.988413353Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.988426933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.988441052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} {"msg":"Crucible b8f011b1-8113-47da-8b9f-44e79f9fffa9 has session id: 06641fa6-1d02-407d-b362-61e68f1518a1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.988964709Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Upstairs opts: Upstairs UUID: b8f011b1-8113-47da-8b9f-44e79f9fffa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989008947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Crucible stats registered with UUID: b8f011b1-8113-47da-8b9f-44e79f9fffa9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989032967Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:07.989053246Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989465505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989491745Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} {"msg":"b8f011b1-8113-47da-8b9f-44e79f9fffa9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989550683Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989567833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"0","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989591102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"0","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989605302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"1","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989619611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"1","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989632981Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"2","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.98964838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"2","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.9896698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989765817Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989789427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:41181)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989815336Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44365)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989839735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:60185)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.989876205Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.990658934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"1","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.990687723Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"2","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.990702603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"0","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.991364425Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"1","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.991405274Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"2","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.991435713Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"0","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992028768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992054847Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[0]R flush_numbers[0..12]: [223, 211, 225, 225, 215, 216, 225, 224, 227, 227, 221, 222]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992069177Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[0]R generation[0..12]: [118, 114, 120, 120, 114, 115, 120, 119, 121, 121, 117, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992083926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992098096Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[1]R flush_numbers[0..12]: [223, 211, 225, 225, 215, 216, 225, 224, 227, 227, 221, 222]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992111486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[1]R generation[0..12]: [118, 114, 120, 120, 114, 115, 120, 119, 121, 121, 117, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992125565Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992139505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[2]R flush_numbers[0..12]: [223, 211, 225, 225, 215, 216, 225, 224, 227, 227, 221, 222]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992156465Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[2]R generation[0..12]: [118, 114, 120, 120, 114, 115, 120, 119, 121, 121, 117, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992170314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992183834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992196473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992209313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992221943Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992234752Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"":"downstairs","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992247602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992259162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"0","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992271911Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"1","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992284761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"client":"2","":"downstairs","session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992297821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992309761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"b8f011b1-8113-47da-8b9f-44e79f9fffa9 is now active with session: 06641fa6-1d02-407d-b362-61e68f1518a1","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:07:07.99232209Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.99234154Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680,"session_id":"06641fa6-1d02-407d-b362-61e68f1518a1"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992355339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} {"msg":"Activated sub_volume b8f011b1-8113-47da-8b9f-44e79f9fffa9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:07.992366589Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 449 len 16384 data: 4 7 8 9 02/30 Read block 102 len 4096 03/30 Write block 455 len 24576 data: 10 8 8 9 8 6 04/30 Write block 337 len 32768 data: 14 16 14 16 16 16 14 17 05/30 Read block 372 len 32768 06/30 Flush 07/30 Write block 41 len 8192 data: 8 9 08/30 Write block 370 len 16384 data: 11 11 11 11 09/30 Write block 577 len 4096 data: 11 10/30 Write block 248 len 4096 data: 15 11/30 Write block 395 len 32768 data: 15 12 14 17 14 15 14 11 12/30 Read block 371 len 8192 13/30 Read block 588 len 36864 14/30 Read block 279 len 8192 15/30 Write block 2 len 4096 data: 2 16/30 Write block 566 len 28672 data: 12 11 12 10 10 8 10 17/30 Write block 162 len 32768 data: 7 6 7 6 9 7 7 7 18/30 Write block 372 len 36864 data: 12 12 13 10 9 12 14 15 15 19/30 Write block 314 len 32768 data: 10 8 9 9 7 8 8 8 20/30 Write block 212 len 24576 data: 10 8 8 8 8 9 21/30 Write block 201 len 28672 data: 8 9 7 7 8 7 6 22/30 Read block 78 len 4096 23/30 Read block 117 len 8192 24/30 Read block 253 len 20480 25/30 Read block 318 len 20480 26/30 Write block 251 len 20480 data: 13 13 10 9 8 27/30 Read block 347 len 16384 28/30 Write block 97 len 24576 data: 14 14 14 15 16 15 29/30 Read block 548 len 32768 30/30 Write block 225 len 24576 data: 12 13 13 10 8 7 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 2 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:08.011931453Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 223 223 223 T T T 1 020-039 114 114 114 211 211 211 F F F 2 040-059 120 120 120 225 225 225 T T T 3 060-079 120 120 120 225 225 225 F F F 4 080-099 114 114 114 215 215 215 T T T 5 100-119 115 115 115 216 216 216 T T T 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 121 121 121 227 227 227 T T T 9 180-199 121 121 121 227 227 227 F F F 10 200-219 117 117 117 221 221 221 T T T 11 220-239 118 118 118 222 222 222 T T T 12 240-259 120 120 120 226 226 226 T T T 13 260-279 120 120 120 226 226 226 F F F 14 280-299 120 120 120 226 226 226 F F F 15 300-319 120 120 120 225 225 225 T T T 16 320-339 122 122 122 228 228 228 T T T 17 340-359 122 122 122 228 228 228 F F F 18 360-379 121 121 121 227 227 227 T T T 19 380-399 121 121 121 227 227 227 T T T 20 400-419 121 121 121 227 227 227 T T T 21 420-439 112 112 112 207 207 207 F F F 22 440-459 122 122 122 228 228 228 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 119 119 119 224 224 224 T T T 29 580-599 106 106 106 198 198 198 F F F Max gen: 122, Max flush: 228 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 229 229 229 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 123 123 123 229 229 229 F F F 5 100-119 123 123 123 229 229 229 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 123 123 123 229 229 229 F F F 9 180-199 121 121 121 227 227 227 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 123 123 123 229 229 229 F F F 12 240-259 123 123 123 229 229 229 F F F 13 260-279 120 120 120 226 226 226 F F F 14 280-299 120 120 120 226 226 226 F F F 15 300-319 123 123 123 229 229 229 F F F 16 320-339 123 123 123 229 229 229 F F F 17 340-359 122 122 122 228 228 228 F F F 18 360-379 123 123 123 229 229 229 F F F 19 380-399 123 123 123 229 229 229 F F F 20 400-419 123 123 123 229 229 229 F F F 21 420-439 112 112 112 207 207 207 F F F 22 440-459 122 122 122 228 228 228 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 116 116 116 218 218 218 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 123 123 123 229 229 229 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 123, Max flush: 229 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.321745173Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:12.364218074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.364628303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.39548621Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.396073195Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.396108114Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.396121543Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.396135683Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} {"msg":"Crucible 661e1919-b27a-43aa-9991-2a8ba301fde0 has session id: a8990bcc-22c2-45a0-bfa7-63ba594a2d4f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.396814385Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Upstairs opts: Upstairs UUID: 661e1919-b27a-43aa-9991-2a8ba301fde0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.396847934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Crucible stats registered with UUID: 661e1919-b27a-43aa-9991-2a8ba301fde0","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.396869404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:12.396883313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.39737578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.39739912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} {"msg":"661e1919-b27a-43aa-9991-2a8ba301fde0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397466298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397483937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"0","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397513247Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"0","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397538926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"1","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397566475Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"1","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397592665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"2","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397617074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"2","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397635893Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397661033Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397696302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50007)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397732461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50317)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.39776219Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45968)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.397892467Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.398409833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"2","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.398456202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"1","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.398566169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"0","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399102165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"1","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399131384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"2","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399147654Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"0","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399720369Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399745768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[0]R flush_numbers[0..12]: [229, 211, 229, 225, 229, 229, 225, 224, 229, 227, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399786577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[0]R generation[0..12]: [123, 114, 123, 120, 123, 123, 120, 119, 123, 121, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399804356Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399818596Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[1]R flush_numbers[0..12]: [229, 211, 229, 225, 229, 229, 225, 224, 229, 227, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399832176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[1]R generation[0..12]: [123, 114, 123, 120, 123, 123, 120, 119, 123, 121, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399845865Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399859645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[2]R flush_numbers[0..12]: [229, 211, 229, 225, 229, 229, 225, 224, 229, 227, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399873165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[2]R generation[0..12]: [123, 114, 123, 120, 123, 123, 120, 119, 123, 121, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399886254Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399899804Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399915644Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399929013Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399941733Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399954112Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"":"downstairs","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399966912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399978602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"0","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.399991482Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"1","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.400004431Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"client":"2","":"downstairs","session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.40002405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.40003611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"661e1919-b27a-43aa-9991-2a8ba301fde0 is now active with session: a8990bcc-22c2-45a0-bfa7-63ba594a2d4f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.40004852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.40006006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688,"session_id":"a8990bcc-22c2-45a0-bfa7-63ba594a2d4f"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.400072209Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} {"msg":"Activated sub_volume 661e1919-b27a-43aa-9991-2a8ba301fde0","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:12.400083169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 338 len 24576 data: 17 15 17 17 17 15 02/30 Read block 335 len 8192 03/30 Write block 522 len 36864 data: 8 9 9 9 7 9 10 10 8 04/30 Read block 152 len 32768 05/30 Read block 489 len 16384 06/30 Flush 07/30 Write block 175 len 4096 data: 12 08/30 Read block 522 len 24576 09/30 Write block 266 len 16384 data: 13 14 14 14 10/30 Write block 253 len 4096 data: 11 11/30 Read block 572 len 24576 12/30 Read block 420 len 12288 13/30 Write block 422 len 20480 data: 10 11 11 10 7 14/30 Read block 365 len 12288 15/30 Read block 66 len 4096 16/30 Read block 215 len 40960 17/30 Read block 208 len 20480 18/30 Read block 418 len 36864 19/30 Write block 532 len 12288 data: 10 11 11 20/30 Read block 21 len 40960 21/30 Flush 22/30 Flush 23/30 Write block 419 len 12288 data: 13 14 10 24/30 Read block 128 len 28672 25/30 Read block 1 len 36864 26/30 Write block 450 len 20480 data: 8 9 10 10 10 27/30 Write block 176 len 16384 data: 10 8 11 11 28/30 Write block 358 len 8192 data: 8 11 29/30 Read block 88 len 16384 30/30 Read block 292 len 40960 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:12.420895951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 229 229 229 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 123 123 123 229 229 229 F F F 5 100-119 123 123 123 229 229 229 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 124 124 124 231 231 231 T T T 9 180-199 121 121 121 227 227 227 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 123 123 123 229 229 229 F F F 12 240-259 124 124 124 231 231 231 F F F 13 260-279 124 124 124 231 231 231 F F F 14 280-299 120 120 120 226 226 226 F F F 15 300-319 123 123 123 229 229 229 F F F 16 320-339 124 124 124 230 230 230 F F F 17 340-359 124 124 124 230 230 230 T T T 18 360-379 123 123 123 229 229 229 F F F 19 380-399 123 123 123 229 229 229 F F F 20 400-419 123 123 123 229 229 229 T T T 21 420-439 124 124 124 231 231 231 T T T 22 440-459 122 122 122 228 228 228 T T T 23 460-479 122 122 122 228 228 228 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 123 123 123 229 229 229 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 124, Max flush: 231 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 229 229 229 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 123 123 123 229 229 229 F F F 5 100-119 123 123 123 229 229 229 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 125 125 125 232 232 232 F F F 9 180-199 121 121 121 227 227 227 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 123 123 123 229 229 229 F F F 12 240-259 124 124 124 231 231 231 F F F 13 260-279 124 124 124 231 231 231 F F F 14 280-299 120 120 120 226 226 226 F F F 15 300-319 123 123 123 229 229 229 F F F 16 320-339 124 124 124 230 230 230 F F F 17 340-359 125 125 125 232 232 232 F F F 18 360-379 123 123 123 229 229 229 F F F 19 380-399 123 123 123 229 229 229 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 125 125 125 232 232 232 F F F 22 440-459 125 125 125 232 232 232 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 118 118 118 223 223 223 F F F 25 500-519 118 118 118 223 223 223 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 102 102 102 191 191 191 F F F 28 560-579 123 123 123 229 229 229 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 125, Max flush: 232 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.68632135Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.729882682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.730566884Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.730600454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.730613883Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.730627813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} {"msg":"Crucible 1759644f-2e33-493d-bf13-e7a46c389180 has session id: dc518932-a2c7-4201-9325-6480c914d075","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731163019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Upstairs opts: Upstairs UUID: 1759644f-2e33-493d-bf13-e7a46c389180, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731217837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Crucible stats registered with UUID: 1759644f-2e33-493d-bf13-e7a46c389180","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731246917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:16.731269276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731745773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731778853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} {"msg":"1759644f-2e33-493d-bf13-e7a46c389180 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731836221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.73186572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"0","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731908669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"0","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731937328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"1","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731966788Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"1","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.731993337Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"2","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732018426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"2","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732039456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732064735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732287779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"ds_connection connected from Ok(127.0.0.1:40213)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732332828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"ds_connection connected from Ok(127.0.0.1:43129)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732360327Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"ds_connection connected from Ok(127.0.0.1:57574)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732387416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732793176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"2","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.732833155Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"0","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.733346471Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"2","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.733459948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"0","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.733870148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.733984234Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.734650437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"1","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735216222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"1","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[0]R flush_numbers[0..12]: [229, 211, 229, 225, 229, 229, 225, 224, 232, 227, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735736108Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[0]R generation[0..12]: [123, 114, 123, 120, 123, 123, 120, 119, 125, 121, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735755418Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735770147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[1]R flush_numbers[0..12]: [229, 211, 229, 225, 229, 229, 225, 224, 232, 227, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735783987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[1]R generation[0..12]: [123, 114, 123, 120, 123, 123, 120, 119, 125, 121, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735801787Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735815866Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[2]R flush_numbers[0..12]: [229, 211, 229, 225, 229, 229, 225, 224, 232, 227, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735829426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[2]R generation[0..12]: [123, 114, 123, 120, 123, 123, 120, 119, 125, 121, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735842665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735856325Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735868825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735881664Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735894334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735916314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"":"downstairs","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735931883Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735943793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"0","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735956762Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"1","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-19T17:07:16.735969682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"client":"2","":"downstairs","session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.735989242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.736017441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"1759644f-2e33-493d-bf13-e7a46c389180 is now active with session: dc518932-a2c7-4201-9325-6480c914d075","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.73603332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.73604599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696,"session_id":"dc518932-a2c7-4201-9325-6480c914d075"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.73605802Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} {"msg":"Activated sub_volume 1759644f-2e33-493d-bf13-e7a46c389180","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:16.736068899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 36864 data: 14 14 10 9 11 13 12 9 12 02/30 Read block 450 len 24576 03/30 Read block 173 len 36864 04/30 Read block 569 len 28672 05/30 Read block 283 len 28672 06/30 Read block 100 len 28672 07/30 Read block 535 len 4096 08/30 Write block 564 len 32768 data: 13 12 13 12 13 11 11 9 09/30 Read block 299 len 28672 10/30 Write block 559 len 36864 data: 3 4 6 9 11 14 13 14 13 11/30 Write block 283 len 36864 data: 6 6 5 5 4 2 5 7 7 12/30 Read block 101 len 8192 13/30 Write block 491 len 32768 data: 6 5 9 12 14 13 11 9 14/30 Read block 589 len 36864 15/30 Write block 500 len 16384 data: 3 4 3 5 16/30 Flush 17/30 Write block 240 len 8192 data: 8 9 18/30 Read block 584 len 12288 19/30 Read block 280 len 16384 20/30 Write block 182 len 4096 data: 8 21/30 Read block 368 len 28672 22/30 Write block 251 len 12288 data: 14 14 12 23/30 Write block 298 len 16384 data: 8 7 8 8 24/30 Write block 451 len 16384 data: 10 11 11 11 25/30 Read block 478 len 16384 26/30 Read block 238 len 24576 27/30 Read block 118 len 4096 28/30 Write block 452 len 16384 data: 12 12 12 11 29/30 Read block 401 len 36864 30/30 Write block 96 len 40960 data: 14 15 15 15 16 17 16 16 16 17 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:16.75807825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 229 229 229 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 123 123 123 229 229 229 T T T 5 100-119 123 123 123 229 229 229 T T T 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 125 125 125 232 232 232 F F F 9 180-199 121 121 121 227 227 227 T T T 10 200-219 123 123 123 229 229 229 F F F 11 220-239 123 123 123 229 229 229 F F F 12 240-259 124 124 124 231 231 231 T T T 13 260-279 124 124 124 231 231 231 F F F 14 280-299 126 126 126 233 233 233 T T T 15 300-319 123 123 123 229 229 229 T T T 16 320-339 124 124 124 230 230 230 F F F 17 340-359 125 125 125 232 232 232 F F F 18 360-379 126 126 126 233 233 233 F F F 19 380-399 123 123 123 229 229 229 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 125 125 125 232 232 232 F F F 22 440-459 125 125 125 232 232 232 T T T 23 460-479 122 122 122 228 228 228 F F F 24 480-499 126 126 126 233 233 233 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 126 126 126 233 233 233 F F F 28 560-579 126 126 126 233 233 233 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 126, Max flush: 233 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 229 229 229 F F F 1 020-039 114 114 114 211 211 211 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 127 127 127 234 234 234 F F F 5 100-119 127 127 127 234 234 234 F F F 6 120-139 120 120 120 225 225 225 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 125 125 125 232 232 232 F F F 9 180-199 127 127 127 234 234 234 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 123 123 123 229 229 229 F F F 12 240-259 127 127 127 234 234 234 F F F 13 260-279 124 124 124 231 231 231 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 127 127 127 234 234 234 F F F 16 320-339 124 124 124 230 230 230 F F F 17 340-359 125 125 125 232 232 232 F F F 18 360-379 126 126 126 233 233 233 F F F 19 380-399 123 123 123 229 229 229 F F F 20 400-419 125 125 125 232 232 232 F F F 21 420-439 125 125 125 232 232 232 F F F 22 440-459 127 127 127 234 234 234 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 126 126 126 233 233 233 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 126 126 126 233 233 233 F F F 28 560-579 126 126 126 233 233 233 F F F 29 580-599 106 106 106 198 198 198 F F F Max gen: 127, Max flush: 234 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.011072058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:31.053421763Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.053807332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.084751657Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.085362681Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.08539607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.08540994Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.085423829Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} {"msg":"Crucible 31e86ae7-11e3-4e7e-8b14-81c7449aa44a has session id: df7d3d0e-97a5-4def-a52b-21fcac4d0418","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086045363Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Upstairs opts: Upstairs UUID: 31e86ae7-11e3-4e7e-8b14-81c7449aa44a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086079402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Crucible stats registered with UUID: 31e86ae7-11e3-4e7e-8b14-81c7449aa44a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086094462Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:31.086106791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086478642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086502661Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} {"msg":"31e86ae7-11e3-4e7e-8b14-81c7449aa44a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.08655243Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086578419Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"0","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086606958Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"0","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086622298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"1","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086636888Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"1","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086649717Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"2","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086663667Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"2","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086676827Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086690326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086703686Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"ds_connection connected from Ok(127.0.0.1:43575)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086717445Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"ds_connection connected from Ok(127.0.0.1:38174)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086730925Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"ds_connection connected from Ok(127.0.0.1:59464)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.086887591Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.087519934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"1","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.087550863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"2","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.08769974Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"0","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088176327Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"2","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088204886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"1","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088320653Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"0","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088747672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088864389Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[0]R flush_numbers[0..12]: [229, 211, 229, 225, 234, 234, 225, 224, 232, 234, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088926607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[0]R generation[0..12]: [123, 114, 123, 120, 127, 127, 120, 119, 125, 127, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088954486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088970066Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[1]R flush_numbers[0..12]: [229, 211, 229, 225, 234, 234, 225, 224, 232, 234, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088984116Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[1]R generation[0..12]: [123, 114, 123, 120, 127, 127, 120, 119, 125, 127, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.088998115Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089011895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[2]R flush_numbers[0..12]: [229, 211, 229, 225, 234, 234, 225, 224, 232, 234, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089025605Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[2]R generation[0..12]: [123, 114, 123, 120, 127, 127, 120, 119, 125, 127, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089039594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089053634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089070073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089083713Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089096553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089109392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"":"downstairs","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089122192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089133822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"0","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089146731Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"1","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089159421Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"client":"2","":"downstairs","session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089172531Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.08918461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"31e86ae7-11e3-4e7e-8b14-81c7449aa44a is now active with session: df7d3d0e-97a5-4def-a52b-21fcac4d0418","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:07:31.08919736Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.08921494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704,"session_id":"df7d3d0e-97a5-4def-a52b-21fcac4d0418"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089226899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} {"msg":"Activated sub_volume 31e86ae7-11e3-4e7e-8b14-81c7449aa44a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:31.089237929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 312 len 16384 data: 9 9 11 9 02/30 Read block 540 len 4096 03/30 Read block 330 len 16384 04/30 Write block 241 len 4096 data: 10 05/30 Write block 497 len 12288 data: 12 10 5 06/30 Read block 165 len 12288 07/30 Write block 111 len 24576 data: 10 7 7 9 8 7 08/30 Read block 236 len 4096 09/30 Flush 10/30 Write block 232 len 36864 data: 5 5 5 9 9 9 9 7 9 11/30 Write block 134 len 24576 data: 10 9 8 9 11 13 12/30 Write block 168 len 12288 data: 8 8 10 13/30 Read block 114 len 32768 14/30 Flush 15/30 Write block 349 len 36864 data: 12 10 9 7 5 7 7 6 6 16/30 Read block 302 len 28672 17/30 Read block 217 len 36864 18/30 Write block 82 len 8192 data: 11 12 19/30 Read block 299 len 28672 20/30 Read block 572 len 20480 21/30 Write block 18 len 24576 data: 12 11 10 10 10 8 22/30 Write block 597 len 8192 data: 4 3 23/30 Write block 418 len 32768 data: 12 14 15 11 11 12 12 11 24/30 Write block 379 len 12288 data: 16 16 13 25/30 Flush 26/30 Write block 332 len 20480 data: 11 13 10 11 12 27/30 Write block 113 len 12288 data: 8 10 9 28/30 Write block 11 len 36864 data: 9 9 9 10 12 11 11 13 12 29/30 Read block 299 len 12288 30/30 Write block 114 len 16384 data: 11 10 8 9 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:31.109415577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 237 237 237 T T T 1 020-039 128 128 128 237 237 237 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 128 128 128 237 237 237 F F F 5 100-119 128 128 128 235 235 235 T T T 6 120-139 128 128 128 236 236 236 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 127 127 127 234 234 234 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 128 128 128 236 236 236 F F F 13 260-279 124 124 124 231 231 231 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 128 128 128 235 235 235 F F F 16 320-339 124 124 124 230 230 230 T T T 17 340-359 128 128 128 237 237 237 F F F 18 360-379 128 128 128 237 237 237 F F F 19 380-399 128 128 128 237 237 237 F F F 20 400-419 128 128 128 237 237 237 F F F 21 420-439 128 128 128 237 237 237 F F F 22 440-459 127 127 127 234 234 234 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 128 128 128 235 235 235 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 126 126 126 233 233 233 F F F 28 560-579 126 126 126 233 233 233 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 128, Max flush: 237 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 238 238 238 F F F 1 020-039 128 128 128 237 237 237 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 128 128 128 237 237 237 F F F 5 100-119 129 129 129 238 238 238 F F F 6 120-139 128 128 128 236 236 236 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 127 127 127 234 234 234 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 128 128 128 236 236 236 F F F 13 260-279 124 124 124 231 231 231 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 128 128 128 235 235 235 F F F 16 320-339 129 129 129 238 238 238 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 128 128 128 237 237 237 F F F 19 380-399 128 128 128 237 237 237 F F F 20 400-419 128 128 128 237 237 237 F F F 21 420-439 128 128 128 237 237 237 F F F 22 440-459 127 127 127 234 234 234 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 128 128 128 235 235 235 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 126 126 126 233 233 233 F F F 28 560-579 126 126 126 233 233 233 F F F 29 580-599 128 128 128 237 237 237 F F F Max gen: 129, Max flush: 238 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.363067426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.406897861Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.407847816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.407882655Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.407897105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.407911385Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} {"msg":"Crucible 3975e878-b250-463b-a619-8fb0951f42a6 has session id: 4e6761e0-383a-4dfc-9895-57a075e02d8f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.408595447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Upstairs opts: Upstairs UUID: 3975e878-b250-463b-a619-8fb0951f42a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.408644386Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Crucible stats registered with UUID: 3975e878-b250-463b-a619-8fb0951f42a6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.408674945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:35.408699254Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409116423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409138762Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} {"msg":"3975e878-b250-463b-a619-8fb0951f42a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409207791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.4092369Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"0","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409266339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"0","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409281259Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"1","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409295748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"1","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409331677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"2","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409347437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"2","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409367367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409393656Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409422885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34039)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409452684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55632)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.409575761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34926)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.40961913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.410186105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"2","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.410214524Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"0","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.410768219Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"0","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.410906156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"2","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.411391223Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.41149858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.411569438Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"1","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412074265Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"1","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[0]R flush_numbers[0..12]: [238, 237, 229, 225, 237, 238, 236, 224, 236, 234, 229, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412582412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[0]R generation[0..12]: [129, 128, 123, 120, 128, 129, 128, 119, 128, 127, 123, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412601131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412615481Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[1]R flush_numbers[0..12]: [238, 237, 229, 225, 237, 238, 236, 224, 236, 234, 229, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.41262939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[1]R generation[0..12]: [129, 128, 123, 120, 128, 129, 128, 119, 128, 127, 123, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.41264636Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.41266049Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[2]R flush_numbers[0..12]: [238, 237, 229, 225, 237, 238, 236, 224, 236, 234, 229, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412674019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[2]R generation[0..12]: [129, 128, 123, 120, 128, 129, 128, 119, 128, 127, 123, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412687349Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412701049Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412713568Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412726578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412739148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412751807Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"":"downstairs","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412774127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412785786Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"0","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412798626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"1","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412811276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"client":"2","":"downstairs","session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412824085Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412837585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"3975e878-b250-463b-a619-8fb0951f42a6 is now active with session: 4e6761e0-383a-4dfc-9895-57a075e02d8f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412850405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412862164Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712,"session_id":"4e6761e0-383a-4dfc-9895-57a075e02d8f"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412873954Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} {"msg":"Activated sub_volume 3975e878-b250-463b-a619-8fb0951f42a6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:35.412884784Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 12288 data: 6 6 7 02/30 Read block 164 len 20480 03/30 Read block 299 len 40960 04/30 Read block 584 len 36864 05/30 Read block 56 len 40960 06/30 Read block 304 len 4096 07/30 Write block 3 len 36864 data: 3 4 5 8 8 6 10 11 10 08/30 Flush 09/30 Write block 319 len 4096 data: 9 10/30 Write block 321 len 4096 data: 9 11/30 Write block 397 len 16384 data: 15 18 15 16 12/30 Read block 209 len 16384 13/30 Write block 584 len 28672 data: 9 9 7 10 6 6 8 14/30 Read block 542 len 40960 15/30 Write block 261 len 4096 data: 15 16/30 Read block 369 len 40960 17/30 Flush 18/30 Write block 571 len 36864 data: 10 11 10 10 11 11 12 11 10 19/30 Read block 58 len 20480 20/30 Read block 542 len 16384 21/30 Write block 566 len 12288 data: 15 14 14 22/30 Write block 244 len 8192 data: 7 10 23/30 Read block 92 len 36864 24/30 Read block 110 len 40960 25/30 Write block 486 len 12288 data: 10 8 7 26/30 Write block 83 len 36864 data: 13 11 9 10 12 12 15 13 13 27/30 Write block 308 len 8192 data: 13 15 28/30 Read block 375 len 40960 29/30 Read block 497 len 12288 30/30 Write block 436 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:35.433682936Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 239 239 239 F F F 1 020-039 128 128 128 237 237 237 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 128 128 128 237 237 237 T T T 5 100-119 129 129 129 238 238 238 F F F 6 120-139 128 128 128 236 236 236 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 128 128 128 236 236 236 T T T 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 130 130 130 240 240 240 T T T 16 320-339 130 130 130 240 240 240 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 128 128 128 237 237 237 F F F 19 380-399 130 130 130 240 240 240 F F F 20 400-419 130 130 130 240 240 240 F F F 21 420-439 128 128 128 237 237 237 T T T 22 440-459 127 127 127 234 234 234 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 128 128 128 235 235 235 T T T 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 126 126 126 233 233 233 F F F 28 560-579 126 126 126 233 233 233 T T T 29 580-599 130 130 130 240 240 240 F F F Max gen: 130, Max flush: 240 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 239 239 239 F F F 1 020-039 128 128 128 237 237 237 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 131 131 131 241 241 241 F F F 5 100-119 129 129 129 238 238 238 F F F 6 120-139 128 128 128 236 236 236 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 123 123 123 229 229 229 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 131 131 131 241 241 241 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 130 130 130 240 240 240 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 128 128 128 237 237 237 F F F 19 380-399 130 130 130 240 240 240 F F F 20 400-419 130 130 130 240 240 240 F F F 21 420-439 131 131 131 241 241 241 F F F 22 440-459 127 127 127 234 234 234 F F F 23 460-479 122 122 122 228 228 228 F F F 24 480-499 131 131 131 241 241 241 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 126 126 126 233 233 233 F F F 28 560-579 131 131 131 241 241 241 F F F 29 580-599 130 130 130 240 240 240 F F F Max gen: 131, Max flush: 241 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.679319276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:39.721387937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.721762547Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.752589585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.753215299Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.753250358Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.753264028Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.753278117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} {"msg":"Crucible aa1b5545-56f7-4654-887c-a6a62f33655e has session id: ed9d5c56-95c3-4bef-9042-1801c2d26445","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.753825583Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Upstairs opts: Upstairs UUID: aa1b5545-56f7-4654-887c-a6a62f33655e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.753987858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Crucible stats registered with UUID: aa1b5545-56f7-4654-887c-a6a62f33655e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754008948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:39.754022138Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754425047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754454186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} {"msg":"aa1b5545-56f7-4654-887c-a6a62f33655e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754510495Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754528844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"0","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754563683Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"0","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754590843Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"1","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754618872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"1","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754648501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"2","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754675671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"2","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.75470257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754734769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754765448Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"ds_connection connected from Ok(127.0.0.1:60111)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754794007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"ds_connection connected from Ok(127.0.0.1:39733)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754824376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"ds_connection connected from Ok(127.0.0.1:61784)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.754852316Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.755675614Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"2","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.755714303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"1","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.755882749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"0","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.756433964Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"2","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.756470373Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"1","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.756649499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"0","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757115076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757151235Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[0]R flush_numbers[0..12]: [239, 237, 229, 225, 241, 238, 236, 224, 236, 239, 229, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757229253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[0]R generation[0..12]: [130, 128, 123, 120, 131, 129, 128, 119, 128, 130, 123, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757249323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757264122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[1]R flush_numbers[0..12]: [239, 237, 229, 225, 241, 238, 236, 224, 236, 239, 229, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757277792Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[1]R generation[0..12]: [130, 128, 123, 120, 131, 129, 128, 119, 128, 130, 123, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757291631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757305491Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[2]R flush_numbers[0..12]: [239, 237, 229, 225, 241, 238, 236, 224, 236, 239, 229, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757319021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[2]R generation[0..12]: [130, 128, 123, 120, 131, 129, 128, 119, 128, 130, 123, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757332761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.75734628Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.7573633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757386109Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757410318Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757438638Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"":"downstairs","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757459967Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757482017Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"0","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757508726Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"1","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757533965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"client":"2","":"downstairs","session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757560404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757586214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"aa1b5545-56f7-4654-887c-a6a62f33655e is now active with session: ed9d5c56-95c3-4bef-9042-1801c2d26445","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757613273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757637032Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720,"session_id":"ed9d5c56-95c3-4bef-9042-1801c2d26445"} {"msg":"The guest has finished waiting for activation with:132","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757666372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} {"msg":"Activated sub_volume aa1b5545-56f7-4654-887c-a6a62f33655e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:39.757687271Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 24576 data: 14 14 11 13 11 11 02/30 Write block 423 len 16384 data: 13 13 12 8 03/30 Write block 206 len 40960 data: 8 7 6 7 8 9 11 9 9 9 04/30 Write block 389 len 24576 data: 5 7 11 11 11 15 05/30 Write block 129 len 4096 data: 10 06/30 Write block 77 len 32768 data: 12 13 11 9 9 12 14 12 07/30 Read block 235 len 8192 08/30 Read block 1 len 12288 09/30 Read block 266 len 24576 10/30 Read block 90 len 4096 11/30 Read block 149 len 8192 12/30 Write block 464 len 40960 data: 7 5 6 6 4 5 7 6 6 7 13/30 Read block 330 len 4096 14/30 Flush 15/30 Write block 596 len 16384 data: 7 5 4 2 16/30 Read block 274 len 28672 17/30 Write block 481 len 8192 data: 11 9 18/30 Write block 398 len 24576 data: 19 16 17 15 12 12 19/30 Write block 564 len 12288 data: 15 14 16 20/30 Flush 21/30 Write block 73 len 8192 data: 8 11 22/30 Write block 443 len 20480 data: 5 6 8 9 8 23/30 Read block 138 len 32768 24/30 Read block 218 len 40960 25/30 Read block 27 len 32768 26/30 Read block 510 len 16384 27/30 Read block 351 len 40960 28/30 Write block 553 len 36864 data: 12 10 9 6 5 4 4 5 7 29/30 Write block 26 len 12288 data: 9 11 13 30/30 Read block 432 len 16384 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:39.778101442Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 239 239 239 F F F 1 020-039 128 128 128 237 237 237 T T T 2 040-059 123 123 123 229 229 229 F F F 3 060-079 132 132 132 242 242 242 T T T 4 080-099 132 132 132 242 242 242 F F F 5 100-119 129 129 129 238 238 238 F F F 6 120-139 132 132 132 242 242 242 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 132 132 132 242 242 242 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 131 131 131 241 241 241 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 130 130 130 240 240 240 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 128 128 128 237 237 237 F F F 19 380-399 132 132 132 243 243 243 F F F 20 400-419 132 132 132 243 243 243 F F F 21 420-439 132 132 132 242 242 242 F F F 22 440-459 127 127 127 234 234 234 T T T 23 460-479 132 132 132 242 242 242 F F F 24 480-499 132 132 132 243 243 243 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 126 126 126 233 233 233 T T T 28 560-579 132 132 132 243 243 243 T T T 29 580-599 132 132 132 243 243 243 F F F Max gen: 132, Max flush: 243 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 239 239 239 F F F 1 020-039 133 133 133 244 244 244 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 133 133 133 244 244 244 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 129 129 129 238 238 238 F F F 6 120-139 132 132 132 242 242 242 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 132 132 132 242 242 242 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 131 131 131 241 241 241 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 130 130 130 240 240 240 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 128 128 128 237 237 237 F F F 19 380-399 132 132 132 243 243 243 F F F 20 400-419 132 132 132 243 243 243 F F F 21 420-439 132 132 132 242 242 242 F F F 22 440-459 133 133 133 244 244 244 F F F 23 460-479 132 132 132 242 242 242 F F F 24 480-499 132 132 132 243 243 243 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 133 133 133 244 244 244 F F F 28 560-579 133 133 133 244 244 244 F F F 29 580-599 132 132 132 243 243 243 F F F Max gen: 133, Max flush: 244 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.045119439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:44.087154772Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.087641729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.119312014Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.120071574Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.120107204Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.120121003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.120135213Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} {"msg":"Crucible 13a77808-f673-4fb7-91da-fc80c12be0f8 has session id: 69e2cccc-1085-4345-bace-c66965bb7f40","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.120794875Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Upstairs opts: Upstairs UUID: 13a77808-f673-4fb7-91da-fc80c12be0f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.120849284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Crucible stats registered with UUID: 13a77808-f673-4fb7-91da-fc80c12be0f8","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.120941591Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:44.120970551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.12138221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121417299Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} {"msg":"13a77808-f673-4fb7-91da-fc80c12be0f8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121475648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121500077Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"0","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121522886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"0","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121537216Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"1","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121551975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"1","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121565015Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"2","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121580845Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"2","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121594044Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121620684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121644923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:34617)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121669342Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:58062)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121837718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"ds_connection connected from Ok(127.0.0.1:40650)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.121878067Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.12251153Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"1","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.122553659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"2","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.122585328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"0","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.123111624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"2","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.123306579Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"1","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.123480695Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"0","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.123789196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.123873464Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[0]R flush_numbers[0..12]: [239, 244, 229, 244, 242, 238, 242, 224, 236, 239, 242, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124117158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[0]R generation[0..12]: [130, 133, 123, 133, 132, 129, 132, 119, 128, 130, 132, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124144337Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124160047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[1]R flush_numbers[0..12]: [239, 244, 229, 244, 242, 238, 242, 224, 236, 239, 242, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124173726Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[1]R generation[0..12]: [130, 133, 123, 133, 132, 129, 132, 119, 128, 130, 132, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124187236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124200876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[2]R flush_numbers[0..12]: [239, 244, 229, 244, 242, 238, 242, 224, 236, 239, 242, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124214405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[2]R generation[0..12]: [130, 133, 123, 133, 132, 129, 132, 119, 128, 130, 132, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124227585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124241245Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124258314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:07:44.124282154Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124305683Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124319182Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"":"downstairs","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124332122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124343802Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"0","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124356692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"1","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124372091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"client":"2","":"downstairs","session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124385141Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124397091Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"13a77808-f673-4fb7-91da-fc80c12be0f8 is now active with session: 69e2cccc-1085-4345-bace-c66965bb7f40","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.12440948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.12442149Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728,"session_id":"69e2cccc-1085-4345-bace-c66965bb7f40"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.12443345Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} {"msg":"Activated sub_volume 13a77808-f673-4fb7-91da-fc80c12be0f8","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:44.124444309Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 252 len 20480 data: 15 13 10 9 9 02/30 Write block 0 len 12288 data: 2 2 3 03/30 Write block 421 len 8192 data: 12 12 04/30 Read block 542 len 32768 05/30 Read block 60 len 20480 06/30 Flush 07/30 Write block 466 len 32768 data: 7 7 5 6 8 7 7 8 08/30 Write block 99 len 20480 data: 16 17 18 17 17 09/30 Read block 218 len 8192 10/30 Write block 409 len 4096 data: 10 11/30 Read block 354 len 16384 12/30 Read block 555 len 8192 13/30 Read block 588 len 40960 14/30 Write block 586 len 12288 data: 8 11 7 15/30 Read block 452 len 8192 16/30 Flush 17/30 Write block 327 len 28672 data: 7 8 8 8 11 12 14 18/30 Read block 466 len 28672 19/30 Read block 362 len 20480 20/30 Write block 373 len 36864 data: 13 14 11 10 13 15 17 17 14 21/30 Read block 271 len 24576 22/30 Read block 124 len 28672 23/30 Read block 383 len 40960 24/30 Write block 568 len 28672 data: 15 12 12 11 12 11 11 25/30 Write block 89 len 28672 data: 16 15 15 12 14 12 12 26/30 Flush 27/30 Write block 541 len 40960 data: 10 11 14 13 14 13 17 17 15 14 28/30 Read block 140 len 16384 29/30 Write block 122 len 12288 data: 8 8 8 30/30 Read block 305 len 40960 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:44.145435576Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 245 245 245 F F F 1 020-039 133 133 133 244 244 244 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 133 133 133 244 244 244 F F F 4 080-099 134 134 134 247 247 247 F F F 5 100-119 134 134 134 246 246 246 F F F 6 120-139 132 132 132 242 242 242 T T T 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 132 132 132 242 242 242 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 134 134 134 245 245 245 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 134 134 134 247 247 247 F F F 19 380-399 134 134 134 247 247 247 F F F 20 400-419 134 134 134 246 246 246 F F F 21 420-439 134 134 134 245 245 245 F F F 22 440-459 133 133 133 244 244 244 F F F 23 460-479 134 134 134 246 246 246 F F F 24 480-499 132 132 132 243 243 243 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 133 133 133 244 244 244 T T T 28 560-579 134 134 134 247 247 247 F F F 29 580-599 134 134 134 246 246 246 F F F Max gen: 134, Max flush: 247 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 245 245 245 F F F 1 020-039 133 133 133 244 244 244 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 133 133 133 244 244 244 F F F 4 080-099 134 134 134 247 247 247 F F F 5 100-119 134 134 134 246 246 246 F F F 6 120-139 135 135 135 248 248 248 F F F 7 140-159 119 119 119 224 224 224 F F F 8 160-179 128 128 128 236 236 236 F F F 9 180-199 130 130 130 239 239 239 F F F 10 200-219 132 132 132 242 242 242 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 134 134 134 245 245 245 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 134 134 134 247 247 247 F F F 19 380-399 134 134 134 247 247 247 F F F 20 400-419 134 134 134 246 246 246 F F F 21 420-439 134 134 134 245 245 245 F F F 22 440-459 133 133 133 244 244 244 F F F 23 460-479 134 134 134 246 246 246 F F F 24 480-499 132 132 132 243 243 243 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 135 135 135 248 248 248 F F F 28 560-579 134 134 134 247 247 247 F F F 29 580-599 134 134 134 246 246 246 F F F Max gen: 135, Max flush: 248 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.394847937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.437961441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.438631073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.438665302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.438679012Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.438693032Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} {"msg":"Crucible c949bc88-dc87-40c8-ab35-de245a003f6d has session id: eab8ba2d-0ca0-42ee-809e-e0c88618155a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.439424742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Upstairs opts: Upstairs UUID: c949bc88-dc87-40c8-ab35-de245a003f6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.439467061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Crucible stats registered with UUID: c949bc88-dc87-40c8-ab35-de245a003f6d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.439489991Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:48.43951105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.43989161Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.43991864Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} {"msg":"c949bc88-dc87-40c8-ab35-de245a003f6d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.439977928Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.439996567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"0","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440020327Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"0","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440038486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440053046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440066265Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"2","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440084735Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"2","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440108844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440131284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440151133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33948)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440165283Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37021)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440197212Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:48.440354048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:48.440395687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:48.440424826Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440499444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.440977452Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"2","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.441142237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"0","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.441535117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"2","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.441639794Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"0","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.442074693Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:48.442190669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.44146328Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47293)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.441631626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.442737646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.443324641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[0]R flush_numbers[0..12]: [245, 244, 229, 244, 247, 246, 248, 224, 236, 239, 242, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.443920725Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[0]R generation[0..12]: [134, 133, 123, 133, 134, 134, 135, 119, 128, 130, 132, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.443943394Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.443958844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[1]R flush_numbers[0..12]: [245, 244, 229, 244, 247, 246, 248, 224, 236, 239, 242, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.443973064Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[1]R generation[0..12]: [134, 133, 123, 133, 134, 134, 135, 119, 128, 130, 132, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.443987144Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444014493Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[2]R flush_numbers[0..12]: [245, 244, 229, 244, 247, 246, 248, 224, 236, 239, 242, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444033302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[2]R generation[0..12]: [134, 133, 123, 133, 134, 134, 135, 119, 128, 130, 132, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444048242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444062571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444082871Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.44410876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.44412589Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.44413942Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"":"downstairs","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444153039Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444170259Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"0","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444184048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"1","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444197358Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"client":"2","":"downstairs","session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444226687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444239677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"c949bc88-dc87-40c8-ab35-de245a003f6d is now active with session: eab8ba2d-0ca0-42ee-809e-e0c88618155a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444252937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444268426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736,"session_id":"eab8ba2d-0ca0-42ee-809e-e0c88618155a"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444281076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} {"msg":"Activated sub_volume c949bc88-dc87-40c8-ab35-de245a003f6d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:07:58.444292365Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 32768 data: 20 17 18 16 13 13 11 11 02/30 Write block 423 len 28672 data: 14 14 13 9 5 8 8 03/30 Write block 15 len 24576 data: 13 12 12 14 13 11 04/30 Flush 05/30 Write block 206 len 40960 data: 9 8 7 8 9 10 12 10 10 10 06/30 Write block 136 len 4096 data: 9 07/30 Read block 504 len 20480 08/30 Write block 155 len 36864 data: 7 8 10 9 8 6 4 8 7 09/30 Write block 152 len 32768 data: 7 6 6 8 9 11 10 9 10/30 Write block 426 len 32768 data: 10 6 9 9 10 12 8 7 11/30 Read block 205 len 20480 12/30 Read block 350 len 32768 13/30 Write block 389 len 40960 data: 6 8 12 12 12 16 16 13 16 21 14/30 Write block 196 len 8192 data: 8 5 15/30 Read block 255 len 20480 16/30 Flush 17/30 Write block 164 len 20480 data: 8 7 10 8 9 18/30 Write block 465 len 8192 data: 6 8 19/30 Read block 241 len 12288 20/30 Write block 401 len 8192 data: 17 14 21/30 Read block 463 len 4096 22/30 Write block 193 len 12288 data: 7 7 7 23/30 Write block 175 len 40960 data: 13 11 9 12 12 10 8 9 9 10 24/30 Write block 3 len 16384 data: 4 5 6 9 25/30 Read block 361 len 28672 26/30 Read block 147 len 36864 27/30 Write block 433 len 8192 data: 8 7 28/30 Write block 65 len 16384 data: 13 14 13 13 29/30 Read block 426 len 4096 30/30 Read block 246 len 36864 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:07:58.464364077Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 249 249 249 T T T 1 020-039 136 136 136 249 249 249 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 133 133 133 244 244 244 T T T 4 080-099 134 134 134 247 247 247 F F F 5 100-119 134 134 134 246 246 246 F F F 6 120-139 136 136 136 250 250 250 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 136 136 136 250 250 250 T T T 9 180-199 136 136 136 250 250 250 T T T 10 200-219 136 136 136 250 250 250 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 134 134 134 245 245 245 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 134 134 134 247 247 247 F F F 19 380-399 136 136 136 250 250 250 F F F 20 400-419 136 136 136 249 249 249 T T T 21 420-439 136 136 136 250 250 250 T T T 22 440-459 133 133 133 244 244 244 F F F 23 460-479 134 134 134 246 246 246 T T T 24 480-499 132 132 132 243 243 243 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 135 135 135 248 248 248 F F F 28 560-579 134 134 134 247 247 247 F F F 29 580-599 134 134 134 246 246 246 F F F Max gen: 136, Max flush: 250 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 251 251 251 F F F 1 020-039 136 136 136 249 249 249 F F F 2 040-059 123 123 123 229 229 229 F F F 3 060-079 137 137 137 251 251 251 F F F 4 080-099 134 134 134 247 247 247 F F F 5 100-119 134 134 134 246 246 246 F F F 6 120-139 136 136 136 250 250 250 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 137 137 137 251 251 251 F F F 10 200-219 136 136 136 250 250 250 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 134 134 134 245 245 245 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 131 131 131 241 241 241 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 134 134 134 247 247 247 F F F 19 380-399 136 136 136 250 250 250 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 133 133 133 244 244 244 F F F 23 460-479 137 137 137 251 251 251 F F F 24 480-499 132 132 132 243 243 243 F F F 25 500-519 126 126 126 233 233 233 F F F 26 520-539 124 124 124 231 231 231 F F F 27 540-559 135 135 135 248 248 248 F F F 28 560-579 134 134 134 247 247 247 F F F 29 580-599 134 134 134 246 246 246 F F F Max gen: 137, Max flush: 251 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.718089185Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:12.760860129Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.761274518Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.792701699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.793291254Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.793325293Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.793338572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.793352622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} {"msg":"Crucible c138ca38-848e-44fb-bb79-88a36851cdd3 has session id: 51e98c66-709a-4d16-8ebe-9588531a48c5","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.793860999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Upstairs opts: Upstairs UUID: c138ca38-848e-44fb-bb79-88a36851cdd3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.793895838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Crucible stats registered with UUID: c138ca38-848e-44fb-bb79-88a36851cdd3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.793912067Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:12.793925367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794322796Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794346716Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} {"msg":"c138ca38-848e-44fb-bb79-88a36851cdd3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794406874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794424284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"0","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794448833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"0","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794477572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"1","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794507022Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"1","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794591229Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"2","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794628188Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"2","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794653098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794678977Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794735875Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57692)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794768195Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51285)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794794714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42382)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.794822163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.795372249Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"2","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.795412318Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"1","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.795526755Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"0","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796020131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"1","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796052371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"2","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796152158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"0","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796616646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796710153Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[0]R flush_numbers[0..12]: [251, 249, 229, 251, 247, 246, 250, 250, 251, 251, 250, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796786441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[0]R generation[0..12]: [137, 136, 123, 137, 134, 134, 136, 136, 137, 137, 136, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796805941Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796820221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[1]R flush_numbers[0..12]: [251, 249, 229, 251, 247, 246, 250, 250, 251, 251, 250, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.79683386Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[1]R generation[0..12]: [137, 136, 123, 137, 134, 134, 136, 136, 137, 137, 136, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.79684729Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796861129Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[2]R flush_numbers[0..12]: [251, 249, 229, 251, 247, 246, 250, 250, 251, 251, 250, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796874679Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[2]R generation[0..12]: [137, 136, 123, 137, 134, 134, 136, 136, 137, 137, 136, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796888129Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796901768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796917508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796930998Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796943877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796956637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"":"downstairs","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796969477Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.796981306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"0","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797008185Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"1","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797022805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"client":"2","":"downstairs","session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797035935Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797054564Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"c138ca38-848e-44fb-bb79-88a36851cdd3 is now active with session: 51e98c66-709a-4d16-8ebe-9588531a48c5","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797067314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797079074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744,"session_id":"51e98c66-709a-4d16-8ebe-9588531a48c5"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797091203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} {"msg":"Activated sub_volume c138ca38-848e-44fb-bb79-88a36851cdd3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:12.797102443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 486 len 20480 data: 11 9 8 6 6 02/30 Write block 374 len 16384 data: 15 12 11 14 03/30 Read block 166 len 36864 04/30 Write block 311 len 32768 data: 11 10 10 12 10 10 10 8 05/30 Read block 154 len 24576 06/30 Write block 555 len 36864 data: 10 7 6 5 5 6 8 10 12 07/30 Read block 91 len 36864 08/30 Read block 135 len 24576 09/30 Read block 192 len 36864 10/30 Read block 547 len 16384 11/30 Read block 422 len 32768 12/30 Write block 32 len 40960 data: 14 13 12 13 12 10 7 7 8 9 13/30 Read block 470 len 12288 14/30 Write block 57 len 8192 data: 10 8 15/30 Flush 16/30 Write block 575 len 40960 data: 12 12 13 12 11 9 9 9 8 10 17/30 Write block 497 len 16384 data: 13 11 6 4 18/30 Read block 178 len 20480 19/30 Write block 366 len 28672 data: 12 14 13 10 13 12 13 20/30 Read block 84 len 36864 21/30 Read block 347 len 20480 22/30 Read block 403 len 40960 23/30 Read block 166 len 40960 24/30 Write block 535 len 32768 data: 13 14 13 13 14 11 11 12 25/30 Read block 404 len 20480 26/30 Write block 112 len 24576 data: 8 9 12 11 9 10 27/30 Read block 349 len 4096 28/30 Write block 550 len 28672 data: 15 14 11 13 11 11 8 29/30 Write block 456 len 28672 data: 9 9 10 9 7 8 8 30/30 Read block 119 len 16384 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 10 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 14 14 15 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:12.819314008Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 251 251 251 F F F 1 020-039 138 138 138 252 252 252 F F F 2 040-059 138 138 138 252 252 252 F F F 3 060-079 137 137 137 251 251 251 F F F 4 080-099 134 134 134 247 247 247 F F F 5 100-119 134 134 134 246 246 246 T T T 6 120-139 136 136 136 250 250 250 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 137 137 137 251 251 251 F F F 10 200-219 136 136 136 250 250 250 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 134 134 134 245 245 245 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 138 138 138 252 252 252 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 138 138 138 252 252 252 T T T 19 380-399 136 136 136 250 250 250 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 133 133 133 244 244 244 T T T 23 460-479 137 137 137 251 251 251 T T T 24 480-499 138 138 138 252 252 252 T T T 25 500-519 126 126 126 233 233 233 T T T 26 520-539 124 124 124 231 231 231 T T T 27 540-559 138 138 138 252 252 252 T T T 28 560-579 138 138 138 252 252 252 T T T 29 580-599 134 134 134 246 246 246 T T T Max gen: 138, Max flush: 252 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 251 251 251 F F F 1 020-039 138 138 138 252 252 252 F F F 2 040-059 138 138 138 252 252 252 F F F 3 060-079 137 137 137 251 251 251 F F F 4 080-099 134 134 134 247 247 247 F F F 5 100-119 139 139 139 253 253 253 F F F 6 120-139 136 136 136 250 250 250 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 137 137 137 251 251 251 F F F 9 180-199 137 137 137 251 251 251 F F F 10 200-219 136 136 136 250 250 250 F F F 11 220-239 128 128 128 236 236 236 F F F 12 240-259 134 134 134 245 245 245 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 138 138 138 252 252 252 F F F 16 320-339 134 134 134 247 247 247 F F F 17 340-359 128 128 128 237 237 237 F F F 18 360-379 139 139 139 253 253 253 F F F 19 380-399 136 136 136 250 250 250 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 139 139 139 253 253 253 F F F 23 460-479 139 139 139 253 253 253 F F F 24 480-499 139 139 139 253 253 253 F F F 25 500-519 139 139 139 253 253 253 F F F 26 520-539 139 139 139 253 253 253 F F F 27 540-559 139 139 139 253 253 253 F F F 28 560-579 139 139 139 253 253 253 F F F 29 580-599 139 139 139 253 253 253 F F F Max gen: 139, Max flush: 253 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.115552835Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.158954861Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.159641323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.159677252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.159691602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.159705811Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} {"msg":"Crucible c9c75f4c-10ee-447c-a7fc-c5c6a00bf615 has session id: ab689da7-c364-4252-991a-de9b9849ca5b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.160303086Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Upstairs opts: Upstairs UUID: c9c75f4c-10ee-447c-a7fc-c5c6a00bf615, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.160338425Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Crucible stats registered with UUID: c9c75f4c-10ee-447c-a7fc-c5c6a00bf615","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.160354514Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:17.160367084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.160859411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.160883771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} {"msg":"c9c75f4c-10ee-447c-a7fc-c5c6a00bf615 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.160955089Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.160973138Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"0","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161002217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"0","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161017637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161033036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161047186Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"2","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161061136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"2","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161074106Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161088025Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161106915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52871)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161356738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:17.161425926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52254)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.161468965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:17.161499494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:17.161524114Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.1616599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.162097768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"0","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.162234295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"2","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.162676013Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"0","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.16280183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"2","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.163223379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:17.163337156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.162764744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44994)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.162866631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.164085999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.164892938Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[0]R flush_numbers[0..12]: [251, 252, 252, 251, 247, 253, 250, 250, 251, 251, 250, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165529311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[0]R generation[0..12]: [137, 138, 138, 137, 134, 139, 136, 136, 137, 137, 136, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.16555805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.16557492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[1]R flush_numbers[0..12]: [251, 252, 252, 251, 247, 253, 250, 250, 251, 251, 250, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.16558924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[1]R generation[0..12]: [137, 138, 138, 137, 134, 139, 136, 136, 137, 137, 136, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165602619Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165616299Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[2]R flush_numbers[0..12]: [251, 252, 252, 251, 247, 253, 250, 250, 251, 251, 250, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165629779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[2]R generation[0..12]: [137, 138, 138, 137, 134, 139, 136, 136, 137, 137, 136, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165643008Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165656478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165668878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165681607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165697687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:08:27.165710796Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"":"downstairs","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165729076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165740545Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"0","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165753435Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"1","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165766015Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"client":"2","":"downstairs","session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165778684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165790664Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"c9c75f4c-10ee-447c-a7fc-c5c6a00bf615 is now active with session: ab689da7-c364-4252-991a-de9b9849ca5b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165803084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165814734Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752,"session_id":"ab689da7-c364-4252-991a-de9b9849ca5b"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165826633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} {"msg":"Activated sub_volume c9c75f4c-10ee-447c-a7fc-c5c6a00bf615","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:27.165837423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 573 len 24576 data: 12 12 13 13 14 13 03/30 Read block 541 len 36864 04/30 Read block 328 len 28672 05/30 Write block 168 len 16384 data: 10 9 11 12 06/30 Write block 542 len 16384 data: 13 15 14 15 07/30 Read block 6 len 20480 08/30 Read block 226 len 28672 09/30 Write block 120 len 40960 data: 10 10 9 9 9 10 7 11 11 11 10/30 Flush 11/30 Write block 349 len 36864 data: 13 11 10 8 6 8 8 7 7 12/30 Read block 38 len 28672 13/30 Write block 493 len 24576 data: 10 13 15 14 14 12 14/30 Read block 252 len 12288 15/30 Write block 235 len 8192 data: 10 10 16/30 Read block 39 len 24576 17/30 Write block 178 len 32768 data: 13 13 11 9 10 10 11 9 18/30 Write block 245 len 8192 data: 11 12 19/30 Read block 307 len 12288 20/30 Read block 13 len 36864 21/30 Read block 230 len 4096 22/30 Read block 217 len 8192 23/30 Read block 195 len 12288 24/30 Read block 194 len 24576 25/30 Read block 287 len 32768 26/30 Read block 292 len 16384 27/30 Read block 7 len 20480 28/30 Read block 113 len 40960 29/30 Write block 319 len 12288 data: 10 9 10 30/30 Write block 377 len 12288 data: 15 16 18 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 9 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:27.187627809Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 251 251 251 F F F 1 020-039 138 138 138 252 252 252 F F F 2 040-059 138 138 138 252 252 252 F F F 3 060-079 137 137 137 251 251 251 F F F 4 080-099 134 134 134 247 247 247 F F F 5 100-119 139 139 139 253 253 253 F F F 6 120-139 140 140 140 255 255 255 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 140 140 140 255 255 255 T T T 9 180-199 137 137 137 251 251 251 T T T 10 200-219 136 136 136 250 250 250 F F F 11 220-239 128 128 128 236 236 236 T T T 12 240-259 134 134 134 245 245 245 T T T 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 138 138 138 252 252 252 T T T 16 320-339 134 134 134 247 247 247 T T T 17 340-359 128 128 128 237 237 237 T T T 18 360-379 139 139 139 253 253 253 T T T 19 380-399 136 136 136 250 250 250 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 139 139 139 253 253 253 F F F 23 460-479 139 139 139 253 253 253 F F F 24 480-499 139 139 139 253 253 253 T T T 25 500-519 139 139 139 253 253 253 F F F 26 520-539 139 139 139 253 253 253 F F F 27 540-559 140 140 140 255 255 255 F F F 28 560-579 140 140 140 255 255 255 F F F 29 580-599 139 139 139 253 253 253 F F F Max gen: 140, Max flush: 255 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 251 251 251 F F F 1 020-039 138 138 138 252 252 252 F F F 2 040-059 138 138 138 252 252 252 F F F 3 060-079 137 137 137 251 251 251 F F F 4 080-099 134 134 134 247 247 247 F F F 5 100-119 139 139 139 253 253 253 F F F 6 120-139 140 140 140 255 255 255 F F F 7 140-159 136 136 136 250 250 250 F F F 8 160-179 141 141 141 256 256 256 F F F 9 180-199 141 141 141 256 256 256 F F F 10 200-219 136 136 136 250 250 250 F F F 11 220-239 141 141 141 256 256 256 F F F 12 240-259 141 141 141 256 256 256 F F F 13 260-279 130 130 130 240 240 240 F F F 14 280-299 127 127 127 234 234 234 F F F 15 300-319 141 141 141 256 256 256 F F F 16 320-339 141 141 141 256 256 256 F F F 17 340-359 141 141 141 256 256 256 F F F 18 360-379 141 141 141 256 256 256 F F F 19 380-399 136 136 136 250 250 250 F F F 20 400-419 137 137 137 251 251 251 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 139 139 139 253 253 253 F F F 23 460-479 139 139 139 253 253 253 F F F 24 480-499 141 141 141 256 256 256 F F F 25 500-519 139 139 139 253 253 253 F F F 26 520-539 139 139 139 253 253 253 F F F 27 540-559 140 140 140 255 255 255 F F F 28 560-579 140 140 140 255 255 255 F F F 29 580-599 139 139 139 253 253 253 F F F Max gen: 141, Max flush: 256 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.449691519Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.494024901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.4952073Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.495248419Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.495262928Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.495277158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} {"msg":"Crucible 347deb6b-ac64-4589-b17c-ce8683456578 has session id: 29f86f1f-7a74-4d19-8bf1-ccca150dc8ff","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.495935451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Upstairs opts: Upstairs UUID: 347deb6b-ac64-4589-b17c-ce8683456578, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.49598097Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Crucible stats registered with UUID: 347deb6b-ac64-4589-b17c-ce8683456578","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496004269Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:41.496024968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496440927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496486146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} {"msg":"347deb6b-ac64-4589-b17c-ce8683456578 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496558924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496590383Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"0","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496619133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"0","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496634882Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496660421Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496687211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"2","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.4967191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"2","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496748779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496783378Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496814808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:50676)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496849517Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:37692)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.496877326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:41.497016092Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:41.497055741Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:41.497203847Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.497379633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.497826321Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"2","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.49785395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"0","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.498575651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"0","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.49860296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"2","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.499164346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:41.499261613Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.498382669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:39581)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.498516316Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.499647036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500233571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[0]R flush_numbers[0..12]: [251, 252, 252, 251, 247, 253, 255, 250, 256, 256, 250, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500855294Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[0]R generation[0..12]: [137, 138, 138, 137, 134, 139, 140, 136, 141, 141, 136, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500877814Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500892743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[1]R flush_numbers[0..12]: [251, 252, 252, 251, 247, 253, 255, 250, 256, 256, 250, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500906323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[1]R generation[0..12]: [137, 138, 138, 137, 134, 139, 140, 136, 141, 141, 136, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500919762Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500933642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[2]R flush_numbers[0..12]: [251, 252, 252, 251, 247, 253, 255, 250, 256, 256, 250, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500947072Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[2]R generation[0..12]: [137, 138, 138, 137, 134, 139, 140, 136, 141, 141, 136, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500960251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500973541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.500985971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.50099877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.5010115Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.50102769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"":"downstairs","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501041179Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501052929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"0","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501065698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"1","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501078388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"client":"2","":"downstairs","session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501091508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501103388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"347deb6b-ac64-4589-b17c-ce8683456578 is now active with session: 29f86f1f-7a74-4d19-8bf1-ccca150dc8ff","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501115887Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501127687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760,"session_id":"29f86f1f-7a74-4d19-8bf1-ccca150dc8ff"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501139767Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} {"msg":"Activated sub_volume 347deb6b-ac64-4589-b17c-ce8683456578","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:51.501150716Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 101 len 16384 data: 19 18 18 17 02/30 Flush 03/30 Write block 454 len 28672 data: 13 12 10 10 11 10 8 04/30 Read block 27 len 40960 05/30 Flush 06/30 Write block 241 len 4096 data: 11 07/30 Write block 272 len 40960 data: 12 13 14 15 13 12 10 9 8 7 08/30 Flush 09/30 Write block 522 len 28672 data: 9 10 10 10 8 10 11 10/30 Write block 184 len 4096 data: 12 11/30 Write block 475 len 40960 data: 12 10 11 10 12 12 12 10 8 8 12/30 Flush 13/30 Write block 586 len 4096 data: 9 14/30 Read block 183 len 32768 15/30 Write block 78 len 40960 data: 14 12 10 10 13 15 13 10 11 13 16/30 Read block 382 len 12288 17/30 Read block 456 len 24576 18/30 Read block 395 len 16384 19/30 Write block 281 len 20480 data: 8 6 7 7 6 20/30 Write block 137 len 24576 data: 10 12 14 12 14 15 21/30 Write block 76 len 8192 data: 13 13 22/30 Read block 521 len 36864 23/30 Read block 9 len 24576 24/30 Read block 85 len 32768 25/30 Write block 591 len 16384 data: 7 7 6 7 26/30 Write block 6 len 4096 data: 10 27/30 Write block 392 len 36864 data: 13 13 17 17 14 17 22 18 19 28/30 Read block 115 len 28672 29/30 Write block 26 len 4096 data: 10 30/30 Read block 229 len 28672 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 1 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 17 18 16 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:51.522236821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 251 251 251 T T T 1 020-039 138 138 138 252 252 252 T T T 2 040-059 138 138 138 252 252 252 F F F 3 060-079 137 137 137 251 251 251 T T T 4 080-099 134 134 134 247 247 247 T T T 5 100-119 142 142 142 257 257 257 F F F 6 120-139 140 140 140 255 255 255 T T T 7 140-159 136 136 136 250 250 250 T T T 8 160-179 141 141 141 256 256 256 F F F 9 180-199 142 142 142 260 260 260 F F F 10 200-219 136 136 136 250 250 250 F F F 11 220-239 141 141 141 256 256 256 F F F 12 240-259 142 142 142 259 259 259 F F F 13 260-279 142 142 142 259 259 259 F F F 14 280-299 142 142 142 259 259 259 T T T 15 300-319 141 141 141 256 256 256 F F F 16 320-339 141 141 141 256 256 256 F F F 17 340-359 141 141 141 256 256 256 F F F 18 360-379 141 141 141 256 256 256 F F F 19 380-399 136 136 136 250 250 250 T T T 20 400-419 137 137 137 251 251 251 T T T 21 420-439 137 137 137 251 251 251 F F F 22 440-459 142 142 142 258 258 258 F F F 23 460-479 142 142 142 260 260 260 F F F 24 480-499 142 142 142 260 260 260 F F F 25 500-519 139 139 139 253 253 253 F F F 26 520-539 142 142 142 260 260 260 F F F 27 540-559 140 140 140 255 255 255 F F F 28 560-579 140 140 140 255 255 255 F F F 29 580-599 139 139 139 253 253 253 T T T Max gen: 142, Max flush: 260 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 261 261 261 F F F 1 020-039 143 143 143 261 261 261 F F F 2 040-059 138 138 138 252 252 252 F F F 3 060-079 143 143 143 261 261 261 F F F 4 080-099 143 143 143 261 261 261 F F F 5 100-119 142 142 142 257 257 257 F F F 6 120-139 143 143 143 261 261 261 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 141 141 141 256 256 256 F F F 9 180-199 142 142 142 260 260 260 F F F 10 200-219 136 136 136 250 250 250 F F F 11 220-239 141 141 141 256 256 256 F F F 12 240-259 142 142 142 259 259 259 F F F 13 260-279 142 142 142 259 259 259 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 141 141 141 256 256 256 F F F 16 320-339 141 141 141 256 256 256 F F F 17 340-359 141 141 141 256 256 256 F F F 18 360-379 141 141 141 256 256 256 F F F 19 380-399 143 143 143 261 261 261 F F F 20 400-419 143 143 143 261 261 261 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 142 142 142 258 258 258 F F F 23 460-479 142 142 142 260 260 260 F F F 24 480-499 142 142 142 260 260 260 F F F 25 500-519 139 139 139 253 253 253 F F F 26 520-539 142 142 142 260 260 260 F F F 27 540-559 140 140 140 255 255 255 F F F 28 560-579 140 140 140 255 255 255 F F F 29 580-599 143 143 143 261 261 261 F F F Max gen: 143, Max flush: 261 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.784298366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.827839349Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.828531671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.82856663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.82858034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.828594479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} {"msg":"Crucible 7936ba8c-15b9-40e4-8345-291f0fc5e4ad has session id: 7f7b177b-73bf-40e0-8b75-220f9d283b16","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829153974Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Upstairs opts: Upstairs UUID: 7936ba8c-15b9-40e4-8345-291f0fc5e4ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829191384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Crucible stats registered with UUID: 7936ba8c-15b9-40e4-8345-291f0fc5e4ad","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829210833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:55.829224113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829667821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.82969549Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} {"msg":"7936ba8c-15b9-40e4-8345-291f0fc5e4ad active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829751939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829769118Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"0","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829792818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"0","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829806657Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829820877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829833947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"2","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829847596Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"2","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829860426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829914805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.829942044Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"ds_connection connected from Ok(127.0.0.1:45800)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.830043651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:55.83008239Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:55.830115119Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"ds_connection connected from Ok(127.0.0.1:59871)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.830144188Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:08:55.830171738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.830299554Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.830686004Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"2","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.830721033Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"0","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.831309978Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"0","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.831394535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"2","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.831856903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:08:55.831964791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.830777344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"ds_connection connected from Ok(127.0.0.1:62128)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.83094443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.832033451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.832725652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[0]R flush_numbers[0..12]: [261, 261, 252, 261, 261, 257, 261, 261, 256, 260, 250, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833362316Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[0]R generation[0..12]: [143, 143, 138, 143, 143, 142, 143, 143, 141, 142, 136, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833384905Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833399975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[1]R flush_numbers[0..12]: [261, 261, 252, 261, 261, 257, 261, 261, 256, 260, 250, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833413914Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[1]R generation[0..12]: [143, 143, 138, 143, 143, 142, 143, 143, 141, 142, 136, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833427804Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833441534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[2]R flush_numbers[0..12]: [261, 261, 252, 261, 261, 257, 261, 261, 256, 260, 250, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833455043Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[2]R generation[0..12]: [143, 143, 138, 143, 143, 142, 143, 143, 141, 142, 136, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833468413Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833482192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833494792Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833507642Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833520282Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833532941Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"":"downstairs","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833549741Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.83356175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"0","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.83357484Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"1","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.83358771Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"client":"2","":"downstairs","session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833600669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833612749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"7936ba8c-15b9-40e4-8345-291f0fc5e4ad is now active with session: 7f7b177b-73bf-40e0-8b75-220f9d283b16","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833625099Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833636639Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768,"session_id":"7f7b177b-73bf-40e0-8b75-220f9d283b16"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833648498Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} {"msg":"Activated sub_volume 7936ba8c-15b9-40e4-8345-291f0fc5e4ad","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:05.833659428Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 313 len 20480 data: 11 13 11 11 11 02/30 Read block 481 len 20480 03/30 Read block 482 len 40960 04/30 Write block 333 len 16384 data: 15 11 12 13 05/30 Write block 187 len 36864 data: 8 7 8 7 8 9 8 8 8 06/30 Read block 12 len 12288 07/30 Read block 219 len 16384 08/30 Read block 441 len 28672 09/30 Read block 339 len 32768 10/30 Write block 412 len 28672 data: 11 12 8 9 11 11 13 11/30 Read block 315 len 8192 12/30 Write block 406 len 20480 data: 10 12 13 11 13 13/30 Flush 14/30 Write block 177 len 12288 data: 10 14 14 15/30 Write block 555 len 24576 data: 12 9 7 6 6 7 16/30 Write block 487 len 32768 data: 10 9 7 7 7 6 11 14 17/30 Flush 18/30 Write block 389 len 28672 data: 7 9 13 14 14 18 18 19/30 Write block 584 len 40960 data: 11 10 10 12 8 7 9 8 8 7 20/30 Write block 113 len 32768 data: 10 13 12 10 11 9 11 11 21/30 Read block 111 len 16384 22/30 Read block 486 len 40960 23/30 Read block 49 len 20480 24/30 Flush 25/30 Write block 206 len 32768 data: 10 9 8 9 10 11 13 11 26/30 Write block 127 len 16384 data: 12 12 12 8 27/30 Read block 50 len 28672 28/30 Read block 353 len 36864 29/30 Write block 307 len 20480 data: 13 14 16 13 12 30/30 Read block 50 len 40960 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 5 4 6 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:05.856674812Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1768} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 261 261 261 F F F 1 020-039 143 143 143 261 261 261 F F F 2 040-059 138 138 138 252 252 252 F F F 3 060-079 143 143 143 261 261 261 F F F 4 080-099 143 143 143 261 261 261 F F F 5 100-119 144 144 144 264 264 264 F F F 6 120-139 144 144 144 264 264 264 T T T 7 140-159 143 143 143 261 261 261 F F F 8 160-179 144 144 144 263 263 263 F F F 9 180-199 144 144 144 262 262 262 F F F 10 200-219 136 136 136 250 250 250 T T T 11 220-239 141 141 141 256 256 256 F F F 12 240-259 142 142 142 259 259 259 F F F 13 260-279 142 142 142 259 259 259 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 144 144 144 262 262 262 T T T 16 320-339 144 144 144 262 262 262 F F F 17 340-359 141 141 141 256 256 256 F F F 18 360-379 141 141 141 256 256 256 F F F 19 380-399 144 144 144 264 264 264 F F F 20 400-419 144 144 144 262 262 262 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 142 142 142 258 258 258 F F F 23 460-479 142 142 142 260 260 260 F F F 24 480-499 144 144 144 263 263 263 F F F 25 500-519 139 139 139 253 253 253 F F F 26 520-539 142 142 142 260 260 260 F F F 27 540-559 144 144 144 263 263 263 F F F 28 560-579 144 144 144 263 263 263 F F F 29 580-599 144 144 144 264 264 264 F F F Max gen: 144, Max flush: 264 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 261 261 261 F F F 1 020-039 143 143 143 261 261 261 F F F 2 040-059 138 138 138 252 252 252 F F F 3 060-079 143 143 143 261 261 261 F F F 4 080-099 143 143 143 261 261 261 F F F 5 100-119 144 144 144 264 264 264 F F F 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 144 144 144 263 263 263 F F F 9 180-199 144 144 144 262 262 262 F F F 10 200-219 145 145 145 265 265 265 F F F 11 220-239 141 141 141 256 256 256 F F F 12 240-259 142 142 142 259 259 259 F F F 13 260-279 142 142 142 259 259 259 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 145 145 145 265 265 265 F F F 16 320-339 144 144 144 262 262 262 F F F 17 340-359 141 141 141 256 256 256 F F F 18 360-379 141 141 141 256 256 256 F F F 19 380-399 144 144 144 264 264 264 F F F 20 400-419 144 144 144 262 262 262 F F F 21 420-439 137 137 137 251 251 251 F F F 22 440-459 142 142 142 258 258 258 F F F 23 460-479 142 142 142 260 260 260 F F F 24 480-499 144 144 144 263 263 263 F F F 25 500-519 139 139 139 253 253 253 F F F 26 520-539 142 142 142 260 260 260 F F F 27 540-559 144 144 144 263 263 263 F F F 28 560-579 144 144 144 263 263 263 F F F 29 580-599 144 144 144 264 264 264 F F F Max gen: 145, Max flush: 265 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.088927145Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:10.131278789Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.131730597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.162861497Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.16353085Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.163565859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.163579488Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.163593538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} {"msg":"Crucible 0d21f95c-ad77-432e-8495-4b8b3e8e10f9 has session id: 52893271-9202-4958-b71d-1604096fe3e7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164170123Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Upstairs opts: Upstairs UUID: 0d21f95c-ad77-432e-8495-4b8b3e8e10f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164214332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Crucible stats registered with UUID: 0d21f95c-ad77-432e-8495-4b8b3e8e10f9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164238971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:10.16426395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164679299Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164702859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} {"msg":"0d21f95c-ad77-432e-8495-4b8b3e8e10f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164787237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164805036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"0","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164827436Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"0","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164840885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"1","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164854655Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"1","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164867374Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"2","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164880864Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"2","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164903214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164927253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164951332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46908)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164975981Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33329)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.164999611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43916)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.16503507Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.165848989Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"1","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.165889828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"2","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.1661609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"0","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.166614738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"1","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.166658707Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"2","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.166738825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"0","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167206613Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.16730759Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[0]R flush_numbers[0..12]: [261, 261, 252, 261, 261, 264, 265, 261, 263, 262, 265, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167376048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[0]R generation[0..12]: [143, 143, 138, 143, 143, 144, 145, 143, 144, 144, 145, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167395008Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167409348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[1]R flush_numbers[0..12]: [261, 261, 252, 261, 261, 264, 265, 261, 263, 262, 265, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167423007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[1]R generation[0..12]: [143, 143, 138, 143, 143, 144, 145, 143, 144, 144, 145, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167445747Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167476806Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[2]R flush_numbers[0..12]: [261, 261, 252, 261, 261, 264, 265, 261, 263, 262, 265, 256]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167504885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[2]R generation[0..12]: [143, 143, 138, 143, 143, 144, 145, 143, 144, 144, 145, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167530634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167557973Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167588113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167613712Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167638961Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167658181Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"":"downstairs","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.16767145Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.16769112Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"0","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167715459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"1","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167741609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"client":"2","":"downstairs","session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167759648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167772088Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"0d21f95c-ad77-432e-8495-4b8b3e8e10f9 is now active with session: 52893271-9202-4958-b71d-1604096fe3e7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167793497Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167821166Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776,"session_id":"52893271-9202-4958-b71d-1604096fe3e7"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167842066Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} {"msg":"Activated sub_volume 0d21f95c-ad77-432e-8495-4b8b3e8e10f9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:10.167860005Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 59 len 8192 data: 10 10 03/30 Write block 434 len 4096 data: 8 04/30 Read block 211 len 24576 05/30 Write block 221 len 16384 data: 8 9 10 14 06/30 Read block 9 len 16384 07/30 Write block 267 len 16384 data: 15 15 15 16 08/30 Write block 394 len 32768 data: 19 19 15 18 23 19 20 18 09/30 Write block 315 len 36864 data: 12 12 12 9 11 10 11 8 10 10/30 Read block 445 len 28672 11/30 Write block 275 len 20480 data: 16 14 13 11 10 12/30 Read block 469 len 4096 13/30 Read block 122 len 20480 14/30 Read block 140 len 12288 15/30 Write block 354 len 32768 data: 9 9 8 8 9 12 12 12 16/30 Write block 512 len 28672 data: 11 10 12 9 9 9 10 17/30 Write block 474 len 8192 data: 8 13 18/30 Write block 187 len 8192 data: 9 8 19/30 Read block 293 len 36864 20/30 Write block 57 len 24576 data: 11 9 11 11 8 12 21/30 Read block 366 len 12288 22/30 Read block 219 len 40960 23/30 Flush 24/30 Write block 58 len 12288 data: 10 12 12 25/30 Write block 489 len 8192 data: 8 8 26/30 Write block 100 len 4096 data: 18 27/30 Write block 316 len 32768 data: 13 13 10 12 11 12 9 11 28/30 Read block 83 len 40960 29/30 Write block 450 len 12288 data: 9 11 13 30/30 Write block 597 len 8192 data: 6 5 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:10.187556677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1776} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 261 261 261 F F F 1 020-039 143 143 143 261 261 261 F F F 2 040-059 146 146 146 267 267 267 T T T 3 060-079 146 146 146 267 267 267 T T T 4 080-099 143 143 143 261 261 261 F F F 5 100-119 144 144 144 264 264 264 T T T 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 144 144 144 263 263 263 F F F 9 180-199 146 146 146 267 267 267 F F F 10 200-219 145 145 145 265 265 265 F F F 11 220-239 146 146 146 267 267 267 F F F 12 240-259 142 142 142 259 259 259 F F F 13 260-279 146 146 146 267 267 267 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 146 146 146 267 267 267 T T T 16 320-339 146 146 146 267 267 267 T T T 17 340-359 146 146 146 267 267 267 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 146 146 146 267 267 267 F F F 21 420-439 146 146 146 267 267 267 F F F 22 440-459 142 142 142 258 258 258 T T T 23 460-479 146 146 146 267 267 267 F F F 24 480-499 144 144 144 263 263 263 T T T 25 500-519 146 146 146 267 267 267 F F F 26 520-539 142 142 142 260 260 260 F F F 27 540-559 144 144 144 263 263 263 F F F 28 560-579 144 144 144 263 263 263 F F F 29 580-599 144 144 144 264 264 264 T T T Max gen: 146, Max flush: 267 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 261 261 261 F F F 1 020-039 143 143 143 261 261 261 F F F 2 040-059 147 147 147 268 268 268 F F F 3 060-079 147 147 147 268 268 268 F F F 4 080-099 143 143 143 261 261 261 F F F 5 100-119 147 147 147 268 268 268 F F F 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 144 144 144 263 263 263 F F F 9 180-199 146 146 146 267 267 267 F F F 10 200-219 145 145 145 265 265 265 F F F 11 220-239 146 146 146 267 267 267 F F F 12 240-259 142 142 142 259 259 259 F F F 13 260-279 146 146 146 267 267 267 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 147 147 147 268 268 268 F F F 16 320-339 147 147 147 268 268 268 F F F 17 340-359 146 146 146 267 267 267 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 146 146 146 267 267 267 F F F 21 420-439 146 146 146 267 267 267 F F F 22 440-459 147 147 147 268 268 268 F F F 23 460-479 146 146 146 267 267 267 F F F 24 480-499 147 147 147 268 268 268 F F F 25 500-519 146 146 146 267 267 267 F F F 26 520-539 142 142 142 260 260 260 F F F 27 540-559 144 144 144 263 263 263 F F F 28 560-579 144 144 144 263 263 263 F F F 29 580-599 147 147 147 268 268 268 F F F Max gen: 147, Max flush: 268 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.471375321Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.514856445Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.515537767Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.515571277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.515588836Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.515603556Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} {"msg":"Crucible 987de269-6a60-4f48-9711-ac28b07b1a28 has session id: d2fffb26-02d8-433c-b363-c432516e1bd1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516147901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Upstairs opts: Upstairs UUID: 987de269-6a60-4f48-9711-ac28b07b1a28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.51620273Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Crucible stats registered with UUID: 987de269-6a60-4f48-9711-ac28b07b1a28","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516221769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:14.516234639Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516723946Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516749266Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} {"msg":"987de269-6a60-4f48-9711-ac28b07b1a28 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516802384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516828784Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"0","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516853323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"0","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516867512Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"1","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516882052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"1","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516895472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516912571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.516928831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.51695156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.51697482Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48887)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.517032778Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:14.517070537Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:14.517100626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37886)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.517129365Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:14.517161855Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.517362649Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.517804348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"1","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.517832417Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"0","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.51846927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"0","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.51849708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"1","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.519090504Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:14.519109713Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.518137122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53639)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.518296768Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.519425668Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520059271Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[0]R flush_numbers[0..12]: [261, 261, 268, 268, 261, 268, 265, 261, 263, 267, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520656056Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[0]R generation[0..12]: [143, 143, 147, 147, 143, 147, 145, 143, 144, 146, 145, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520691595Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520712454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[1]R flush_numbers[0..12]: [261, 261, 268, 268, 261, 268, 265, 261, 263, 267, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520730114Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[1]R generation[0..12]: [143, 143, 147, 147, 143, 147, 145, 143, 144, 146, 145, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520746783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520764083Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[2]R flush_numbers[0..12]: [261, 261, 268, 268, 261, 268, 265, 261, 263, 267, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520781042Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[2]R generation[0..12]: [143, 143, 147, 147, 143, 147, 145, 143, 144, 146, 145, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520797842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520819601Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520835951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.52085201Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520868Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.52088389Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"":"downstairs","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520899849Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520914439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"0","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520930868Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"1","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520943558Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"client":"2","":"downstairs","session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520956357Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520968117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"987de269-6a60-4f48-9711-ac28b07b1a28 is now active with session: d2fffb26-02d8-433c-b363-c432516e1bd1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.520980577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:09:24.520992167Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784,"session_id":"d2fffb26-02d8-433c-b363-c432516e1bd1"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.521008386Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} {"msg":"Activated sub_volume 987de269-6a60-4f48-9711-ac28b07b1a28","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:24.521019586Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 176 len 16384 data: 12 11 15 15 02/30 Read block 407 len 20480 03/30 Flush 04/30 Write block 24 len 40960 data: 8 9 11 12 14 14 14 15 15 14 05/30 Write block 306 len 36864 data: 13 14 15 17 14 13 11 12 14 06/30 Read block 529 len 4096 07/30 Write block 228 len 12288 data: 11 9 8 08/30 Flush 09/30 Write block 483 len 28672 data: 9 9 8 12 11 10 9 10/30 Read block 516 len 4096 11/30 Read block 70 len 32768 12/30 Read block 594 len 24576 13/30 Write block 378 len 4096 data: 17 14/30 Write block 40 len 40960 data: 9 10 10 11 11 11 8 9 7 7 15/30 Flush 16/30 Write block 535 len 16384 data: 14 15 14 14 17/30 Read block 213 len 8192 18/30 Read block 206 len 24576 19/30 Read block 285 len 8192 20/30 Write block 392 len 24576 data: 15 15 20 20 16 19 21/30 Write block 324 len 36864 data: 10 9 10 8 9 9 9 12 13 22/30 Flush 23/30 Write block 471 len 4096 data: 8 24/30 Flush 25/30 Write block 503 len 4096 data: 6 26/30 Flush 27/30 Write block 114 len 20480 data: 14 13 11 12 10 28/30 Write block 312 len 24576 data: 12 13 15 13 14 14 29/30 Write block 196 len 4096 data: 9 30/30 Write block 237 len 20480 data: 10 10 8 10 12 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:24.540731457Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 261 261 261 F F F 1 020-039 148 148 148 270 270 270 F F F 2 040-059 148 148 148 271 271 271 F F F 3 060-079 147 147 147 268 268 268 F F F 4 080-099 143 143 143 261 261 261 F F F 5 100-119 147 147 147 268 268 268 T T T 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 148 148 148 269 269 269 F F F 9 180-199 146 146 146 267 267 267 T T T 10 200-219 145 145 145 265 265 265 F F F 11 220-239 148 148 148 270 270 270 T T T 12 240-259 142 142 142 259 259 259 T T T 13 260-279 146 146 146 267 267 267 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 148 148 148 270 270 270 T T T 16 320-339 148 148 148 272 272 272 F F F 17 340-359 146 146 146 267 267 267 F F F 18 360-379 148 148 148 271 271 271 F F F 19 380-399 148 148 148 272 272 272 F F F 20 400-419 146 146 146 267 267 267 F F F 21 420-439 146 146 146 267 267 267 F F F 22 440-459 147 147 147 268 268 268 F F F 23 460-479 148 148 148 273 273 273 F F F 24 480-499 148 148 148 271 271 271 F F F 25 500-519 148 148 148 274 274 274 F F F 26 520-539 148 148 148 272 272 272 F F F 27 540-559 144 144 144 263 263 263 F F F 28 560-579 144 144 144 263 263 263 F F F 29 580-599 147 147 147 268 268 268 F F F Max gen: 148, Max flush: 274 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 261 261 261 F F F 1 020-039 148 148 148 270 270 270 F F F 2 040-059 148 148 148 271 271 271 F F F 3 060-079 147 147 147 268 268 268 F F F 4 080-099 143 143 143 261 261 261 F F F 5 100-119 149 149 149 275 275 275 F F F 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 148 148 148 269 269 269 F F F 9 180-199 149 149 149 275 275 275 F F F 10 200-219 145 145 145 265 265 265 F F F 11 220-239 149 149 149 275 275 275 F F F 12 240-259 149 149 149 275 275 275 F F F 13 260-279 146 146 146 267 267 267 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 149 149 149 275 275 275 F F F 16 320-339 148 148 148 272 272 272 F F F 17 340-359 146 146 146 267 267 267 F F F 18 360-379 148 148 148 271 271 271 F F F 19 380-399 148 148 148 272 272 272 F F F 20 400-419 146 146 146 267 267 267 F F F 21 420-439 146 146 146 267 267 267 F F F 22 440-459 147 147 147 268 268 268 F F F 23 460-479 148 148 148 273 273 273 F F F 24 480-499 148 148 148 271 271 271 F F F 25 500-519 148 148 148 274 274 274 F F F 26 520-539 148 148 148 272 272 272 F F F 27 540-559 144 144 144 263 263 263 F F F 28 560-579 144 144 144 263 263 263 F F F 29 580-599 147 147 147 268 268 268 F F F Max gen: 149, Max flush: 275 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.777530698Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.821062942Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.821779203Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.821815082Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.821830221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.821845281Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} {"msg":"Crucible 101cd0ee-64c0-4b67-b9c6-a90fe4abb4fe has session id: 96c8f8ee-3420-413f-9542-f2c7856e78f9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.822365197Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Upstairs opts: Upstairs UUID: 101cd0ee-64c0-4b67-b9c6-a90fe4abb4fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.822430326Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Crucible stats registered with UUID: 101cd0ee-64c0-4b67-b9c6-a90fe4abb4fe","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.822462815Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:28.822482404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.822944822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.822976521Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} {"msg":"101cd0ee-64c0-4b67-b9c6-a90fe4abb4fe active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.82303655Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823069189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"0","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823100968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"0","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823117557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823132147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823145507Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"2","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823168546Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"2","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823191975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823218135Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823240584Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:43588)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823273113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35145)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823299953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:28.823330992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:28.823361281Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:28.823598385Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.823644454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.824096322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"0","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.824124441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"2","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.824762894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"0","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.824790313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"2","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.825433427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:28.825466555Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.824005027Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37991)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.824079425Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.825118938Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.82579098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[0]R flush_numbers[0..12]: [261, 270, 271, 268, 261, 275, 265, 261, 269, 275, 265, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826414294Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[0]R generation[0..12]: [143, 148, 148, 147, 143, 149, 145, 143, 148, 149, 145, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826443553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826460102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[1]R flush_numbers[0..12]: [261, 270, 271, 268, 261, 275, 265, 261, 269, 275, 265, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826474332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[1]R generation[0..12]: [143, 148, 148, 147, 143, 149, 145, 143, 148, 149, 145, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826488392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826502321Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[2]R flush_numbers[0..12]: [261, 270, 271, 268, 261, 275, 265, 261, 269, 275, 265, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826515981Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[2]R generation[0..12]: [143, 148, 148, 147, 143, 149, 145, 143, 148, 149, 145, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.82652962Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.82654323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.82655606Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.82656905Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826581949Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826598599Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"":"downstairs","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826612068Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826624018Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"0","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826637098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"1","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826650187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"client":"2","":"downstairs","session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826663147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826675197Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"101cd0ee-64c0-4b67-b9c6-a90fe4abb4fe is now active with session: 96c8f8ee-3420-413f-9542-f2c7856e78f9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826687676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826699966Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792,"session_id":"96c8f8ee-3420-413f-9542-f2c7856e78f9"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826711856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} {"msg":"Activated sub_volume 101cd0ee-64c0-4b67-b9c6-a90fe4abb4fe","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:38.826722615Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 542 len 12288 data: 14 16 15 02/30 Write block 253 len 36864 data: 14 11 10 10 10 8 11 11 16 03/30 Read block 495 len 40960 04/30 Write block 86 len 12288 data: 12 14 13 05/30 Write block 12 len 24576 data: 10 10 11 14 13 13 06/30 Read block 272 len 16384 07/30 Read block 457 len 36864 08/30 Read block 93 len 12288 09/30 Write block 402 len 32768 data: 15 14 12 12 11 13 14 12 10/30 Write block 23 len 24576 data: 9 9 10 12 13 15 11/30 Write block 464 len 8192 data: 8 7 12/30 Write block 350 len 4096 data: 12 13/30 Read block 453 len 24576 14/30 Read block 88 len 8192 15/30 Read block 224 len 16384 16/30 Read block 88 len 24576 17/30 Read block 483 len 8192 18/30 Flush 19/30 Write block 482 len 24576 data: 11 10 10 9 13 12 20/30 Flush 21/30 Write block 485 len 40960 data: 10 14 13 11 10 9 8 7 12 15 22/30 Write block 551 len 4096 data: 15 23/30 Write block 427 len 4096 data: 7 24/30 Write block 560 len 8192 data: 8 9 25/30 Write block 537 len 20480 data: 15 15 15 12 12 26/30 Write block 596 len 4096 data: 8 27/30 Write block 115 len 20480 data: 14 12 13 11 12 28/30 Write block 565 len 36864 data: 15 17 15 16 13 13 12 13 13 29/30 Write block 409 len 40960 data: 13 14 13 12 13 9 10 12 12 14 30/30 Write block 510 len 32768 data: 11 14 12 11 13 10 10 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:38.84552843Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1792} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 276 276 276 F F F 1 020-039 150 150 150 276 276 276 F F F 2 040-059 148 148 148 271 271 271 F F F 3 060-079 147 147 147 268 268 268 F F F 4 080-099 150 150 150 276 276 276 F F F 5 100-119 149 149 149 275 275 275 T T T 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 148 148 148 269 269 269 F F F 9 180-199 149 149 149 275 275 275 F F F 10 200-219 145 145 145 265 265 265 F F F 11 220-239 149 149 149 275 275 275 F F F 12 240-259 150 150 150 276 276 276 F F F 13 260-279 150 150 150 276 276 276 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 149 149 149 275 275 275 F F F 16 320-339 148 148 148 272 272 272 F F F 17 340-359 150 150 150 276 276 276 F F F 18 360-379 148 148 148 271 271 271 F F F 19 380-399 148 148 148 272 272 272 F F F 20 400-419 150 150 150 276 276 276 T T T 21 420-439 146 146 146 267 267 267 T T T 22 440-459 147 147 147 268 268 268 F F F 23 460-479 150 150 150 276 276 276 F F F 24 480-499 150 150 150 277 277 277 T T T 25 500-519 148 148 148 274 274 274 T T T 26 520-539 148 148 148 272 272 272 T T T 27 540-559 150 150 150 276 276 276 T T T 28 560-579 144 144 144 263 263 263 T T T 29 580-599 147 147 147 268 268 268 T T T Max gen: 150, Max flush: 277 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 276 276 276 F F F 1 020-039 150 150 150 276 276 276 F F F 2 040-059 148 148 148 271 271 271 F F F 3 060-079 147 147 147 268 268 268 F F F 4 080-099 150 150 150 276 276 276 F F F 5 100-119 151 151 151 278 278 278 F F F 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 148 148 148 269 269 269 F F F 9 180-199 149 149 149 275 275 275 F F F 10 200-219 145 145 145 265 265 265 F F F 11 220-239 149 149 149 275 275 275 F F F 12 240-259 150 150 150 276 276 276 F F F 13 260-279 150 150 150 276 276 276 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 149 149 149 275 275 275 F F F 16 320-339 148 148 148 272 272 272 F F F 17 340-359 150 150 150 276 276 276 F F F 18 360-379 148 148 148 271 271 271 F F F 19 380-399 148 148 148 272 272 272 F F F 20 400-419 151 151 151 278 278 278 F F F 21 420-439 151 151 151 278 278 278 F F F 22 440-459 147 147 147 268 268 268 F F F 23 460-479 150 150 150 276 276 276 F F F 24 480-499 151 151 151 278 278 278 F F F 25 500-519 151 151 151 278 278 278 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 151 151 151 278 278 278 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 151 151 151 278 278 278 F F F Max gen: 151, Max flush: 278 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.111499494Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.157763836Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.158494347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.158527826Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.158541665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.158555765Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} {"msg":"Crucible 7c7eef76-a4dd-46fa-b85e-921088e12b3a has session id: 708daf45-680c-47d1-8cc6-f6357ec7302a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159211937Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Upstairs opts: Upstairs UUID: 7c7eef76-a4dd-46fa-b85e-921088e12b3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159247007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Crucible stats registered with UUID: 7c7eef76-a4dd-46fa-b85e-921088e12b3a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159268236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:43.159294045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159691805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159717774Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} {"msg":"7c7eef76-a4dd-46fa-b85e-921088e12b3a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159776043Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159793202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"0","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159816222Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"0","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159829711Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"1","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159843611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"1","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159856451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.15987578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.15989981Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159925639Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159950728Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56569)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.159974477Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:43.160008376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:43.160044586Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45692)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.160073055Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:43.160104124Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.160204591Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.1606395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"1","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.160686619Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"0","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.161290593Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"0","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.161426689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"1","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.161858708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:43.161941706Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.161311225Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36315)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.16149868Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.162563842Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163195856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[0]R flush_numbers[0..12]: [276, 276, 271, 268, 276, 278, 265, 261, 269, 275, 265, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163767081Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[0]R generation[0..12]: [150, 150, 148, 147, 150, 151, 145, 143, 148, 149, 145, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.16379521Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163811579Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[1]R flush_numbers[0..12]: [276, 276, 271, 268, 276, 278, 265, 261, 269, 275, 265, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163826119Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[1]R generation[0..12]: [150, 150, 148, 147, 150, 151, 145, 143, 148, 149, 145, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163840189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163854198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[2]R flush_numbers[0..12]: [276, 276, 271, 268, 276, 278, 265, 261, 269, 275, 265, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163867648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[2]R generation[0..12]: [150, 150, 148, 147, 150, 151, 145, 143, 148, 149, 145, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163881027Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163894487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163907157Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163919986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163932416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163944786Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"":"downstairs","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163957796Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163969495Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"0","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163985965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"1","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.163999344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"client":"2","":"downstairs","session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.164012454Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.164024404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"7c7eef76-a4dd-46fa-b85e-921088e12b3a is now active with session: 708daf45-680c-47d1-8cc6-f6357ec7302a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.164036854Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.164048493Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800,"session_id":"708daf45-680c-47d1-8cc6-f6357ec7302a"} Wait for a query_work_queue command to finish before sending IO {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.164064943Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} {"msg":"Activated sub_volume 7c7eef76-a4dd-46fa-b85e-921088e12b3a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:09:53.164076172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 78 len 4096 data: 15 02/30 Read block 483 len 36864 03/30 Read block 131 len 16384 04/30 Write block 60 len 36864 data: 13 9 13 14 14 14 15 14 14 05/30 Write block 111 len 32768 data: 11 9 11 15 15 13 14 12 06/30 Read block 106 len 8192 07/30 Flush 08/30 Write block 93 len 4096 data: 15 09/30 Write block 325 len 32768 data: 10 11 9 10 10 10 13 14 10/30 Read block 122 len 16384 11/30 Flush 12/30 Flush 13/30 Write block 44 len 36864 data: 12 12 9 10 8 8 8 8 8 14/30 Write block 416 len 32768 data: 13 13 15 15 16 13 13 15 15/30 Read block 474 len 32768 16/30 Write block 216 len 4096 data: 9 17/30 Read block 232 len 8192 18/30 Write block 352 len 4096 data: 9 19/30 Read block 291 len 16384 20/30 Read block 34 len 32768 21/30 Write block 391 len 32768 data: 14 16 16 21 21 17 20 24 22/30 Write block 423 len 20480 data: 16 15 14 11 8 23/30 Read block 474 len 8192 24/30 Read block 190 len 16384 25/30 Write block 249 len 12288 data: 15 15 15 26/30 Write block 308 len 32768 data: 16 18 15 14 13 14 16 14 27/30 Read block 115 len 20480 28/30 Read block 448 len 20480 29/30 Write block 169 len 16384 data: 10 12 13 10 30/30 Read block 74 len 36864 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 17 18 16 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:09:53.184505794Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1800} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 276 276 276 F F F 1 020-039 150 150 150 276 276 276 F F F 2 040-059 148 148 148 271 271 271 T T T 3 060-079 152 152 152 279 279 279 F F F 4 080-099 152 152 152 280 280 280 F F F 5 100-119 152 152 152 279 279 279 F F F 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 148 148 148 269 269 269 T T T 9 180-199 149 149 149 275 275 275 F F F 10 200-219 145 145 145 265 265 265 T T T 11 220-239 149 149 149 275 275 275 F F F 12 240-259 150 150 150 276 276 276 T T T 13 260-279 150 150 150 276 276 276 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 149 149 149 275 275 275 T T T 16 320-339 152 152 152 280 280 280 F F F 17 340-359 150 150 150 276 276 276 T T T 18 360-379 148 148 148 271 271 271 F F F 19 380-399 148 148 148 272 272 272 T T T 20 400-419 151 151 151 278 278 278 T T T 21 420-439 151 151 151 278 278 278 T T T 22 440-459 147 147 147 268 268 268 F F F 23 460-479 150 150 150 276 276 276 F F F 24 480-499 151 151 151 278 278 278 F F F 25 500-519 151 151 151 278 278 278 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 151 151 151 278 278 278 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 151 151 151 278 278 278 F F F Max gen: 152, Max flush: 280 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 276 276 276 F F F 1 020-039 150 150 150 276 276 276 F F F 2 040-059 153 153 153 281 281 281 F F F 3 060-079 152 152 152 279 279 279 F F F 4 080-099 152 152 152 280 280 280 F F F 5 100-119 152 152 152 279 279 279 F F F 6 120-139 145 145 145 265 265 265 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 153 153 153 281 281 281 F F F 9 180-199 149 149 149 275 275 275 F F F 10 200-219 153 153 153 281 281 281 F F F 11 220-239 149 149 149 275 275 275 F F F 12 240-259 153 153 153 281 281 281 F F F 13 260-279 150 150 150 276 276 276 F F F 14 280-299 143 143 143 261 261 261 F F F 15 300-319 153 153 153 281 281 281 F F F 16 320-339 152 152 152 280 280 280 F F F 17 340-359 153 153 153 281 281 281 F F F 18 360-379 148 148 148 271 271 271 F F F 19 380-399 153 153 153 281 281 281 F F F 20 400-419 153 153 153 281 281 281 F F F 21 420-439 153 153 153 281 281 281 F F F 22 440-459 147 147 147 268 268 268 F F F 23 460-479 150 150 150 276 276 276 F F F 24 480-499 151 151 151 278 278 278 F F F 25 500-519 151 151 151 278 278 278 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 151 151 151 278 278 278 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 151 151 151 278 278 278 F F F Max gen: 153, Max flush: 281 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.4414123Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.484894805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.485666284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.485701634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.485715713Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.485729783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} {"msg":"Crucible 44750607-108a-493c-b2fd-d8d9f669f8a7 has session id: e16842a8-65a0-451b-9cdd-b293c5311ea8","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.486311597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Upstairs opts: Upstairs UUID: 44750607-108a-493c-b2fd-d8d9f669f8a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.486413165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Crucible stats registered with UUID: 44750607-108a-493c-b2fd-d8d9f669f8a7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.486434474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:07.486447624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.48694993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.48698638Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} {"msg":"44750607-108a-493c-b2fd-d8d9f669f8a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487045518Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487072237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"0","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487097367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"0","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487111876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487126456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487139896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"2","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487154595Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"2","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487167885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487182224Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487196224Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:07.487555315Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55204)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487605233Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:07.487636603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36839)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487671082Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:07.487695261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.487707611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.488172748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"0","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.488200618Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"2","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.488768853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"0","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.48886626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"2","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.489285719Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:07.489360947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.488406035Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63237)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.488563751Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.489726441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.490358904Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[0]R flush_numbers[0..12]: [276, 276, 281, 279, 280, 279, 265, 261, 281, 275, 281, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.490944988Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[0]R generation[0..12]: [150, 150, 153, 152, 152, 152, 145, 143, 153, 149, 153, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.490978248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.490995527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[1]R flush_numbers[0..12]: [276, 276, 281, 279, 280, 279, 265, 261, 281, 275, 281, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491009757Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[1]R generation[0..12]: [150, 150, 153, 152, 152, 152, 145, 143, 153, 149, 153, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491023276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491036846Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[2]R flush_numbers[0..12]: [276, 276, 281, 279, 280, 279, 265, 261, 281, 275, 281, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491050236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[2]R generation[0..12]: [150, 150, 153, 152, 152, 152, 145, 143, 153, 149, 153, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491063385Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491077115Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491090234Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491103184Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491115684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491128193Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"":"downstairs","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491140883Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491152493Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"0","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491165403Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"1","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491180012Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"client":"2","":"downstairs","session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491192952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491204871Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"44750607-108a-493c-b2fd-d8d9f669f8a7 is now active with session: e16842a8-65a0-451b-9cdd-b293c5311ea8","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:10:17.491217431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491233611Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808,"session_id":"e16842a8-65a0-451b-9cdd-b293c5311ea8"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.491245641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} {"msg":"Activated sub_volume 44750607-108a-493c-b2fd-d8d9f669f8a7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:17.49125644Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 8192 data: 6 7 02/30 Read block 143 len 12288 03/30 Write block 71 len 28672 data: 8 6 9 12 14 14 14 04/30 Write block 343 len 16384 data: 16 18 16 15 05/30 Write block 116 len 12288 data: 14 15 13 06/30 Write block 513 len 16384 data: 12 14 11 11 07/30 Write block 138 len 40960 data: 13 15 13 15 16 17 15 16 16 15 08/30 Flush 09/30 Write block 447 len 16384 data: 9 6 5 10 10/30 Write block 469 len 32768 data: 7 9 9 8 9 9 14 11 11/30 Flush 12/30 Flush 13/30 Write block 412 len 16384 data: 13 14 10 11 14/30 Read block 95 len 28672 15/30 Write block 27 len 12288 data: 14 16 15 16/30 Flush 17/30 Write block 274 len 24576 data: 15 17 15 14 12 11 18/30 Write block 100 len 8192 data: 19 20 19/30 Read block 235 len 8192 20/30 Write block 155 len 28672 data: 9 10 12 11 10 7 5 21/30 Read block 559 len 36864 22/30 Read block 397 len 12288 23/30 Write block 399 len 8192 data: 20 21 24/30 Read block 471 len 12288 25/30 Write block 547 len 28672 data: 18 18 16 16 16 12 14 26/30 Write block 298 len 36864 data: 9 8 9 9 10 11 14 14 14 27/30 Read block 199 len 32768 28/30 Write block 2 len 4096 data: 4 29/30 Write block 392 len 20480 data: 17 17 22 22 18 30/30 Read block 81 len 36864 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 14 14 12 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:17.51099137Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1808} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 282 282 282 T T T 1 020-039 154 154 154 285 285 285 F F F 2 040-059 153 153 153 281 281 281 F F F 3 060-079 154 154 154 282 282 282 F F F 4 080-099 152 152 152 280 280 280 F F F 5 100-119 154 154 154 282 282 282 T T T 6 120-139 154 154 154 282 282 282 F F F 7 140-159 154 154 154 282 282 282 T T T 8 160-179 153 153 153 281 281 281 T T T 9 180-199 149 149 149 275 275 275 F F F 10 200-219 153 153 153 281 281 281 F F F 11 220-239 149 149 149 275 275 275 F F F 12 240-259 153 153 153 281 281 281 F F F 13 260-279 150 150 150 276 276 276 T T T 14 280-299 143 143 143 261 261 261 T T T 15 300-319 153 153 153 281 281 281 T T T 16 320-339 152 152 152 280 280 280 F F F 17 340-359 154 154 154 282 282 282 F F F 18 360-379 148 148 148 271 271 271 F F F 19 380-399 153 153 153 281 281 281 T T T 20 400-419 154 154 154 285 285 285 T T T 21 420-439 153 153 153 281 281 281 F F F 22 440-459 154 154 154 283 283 283 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 151 151 151 278 278 278 F F F 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 151 151 151 278 278 278 T T T 28 560-579 151 151 151 278 278 278 F F F 29 580-599 151 151 151 278 278 278 F F F Max gen: 154, Max flush: 285 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 286 286 286 F F F 1 020-039 154 154 154 285 285 285 F F F 2 040-059 153 153 153 281 281 281 F F F 3 060-079 154 154 154 282 282 282 F F F 4 080-099 152 152 152 280 280 280 F F F 5 100-119 155 155 155 286 286 286 F F F 6 120-139 154 154 154 282 282 282 F F F 7 140-159 155 155 155 286 286 286 F F F 8 160-179 155 155 155 286 286 286 F F F 9 180-199 149 149 149 275 275 275 F F F 10 200-219 153 153 153 281 281 281 F F F 11 220-239 149 149 149 275 275 275 F F F 12 240-259 153 153 153 281 281 281 F F F 13 260-279 155 155 155 286 286 286 F F F 14 280-299 155 155 155 286 286 286 F F F 15 300-319 155 155 155 286 286 286 F F F 16 320-339 152 152 152 280 280 280 F F F 17 340-359 154 154 154 282 282 282 F F F 18 360-379 148 148 148 271 271 271 F F F 19 380-399 155 155 155 286 286 286 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 153 153 153 281 281 281 F F F 22 440-459 154 154 154 283 283 283 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 151 151 151 278 278 278 F F F 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 151 151 151 278 278 278 F F F Max gen: 155, Max flush: 286 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.781549426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.827489876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.828271985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.828308724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.828322974Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.828337183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} {"msg":"Crucible 4314e392-46f2-4562-9f68-ef3de23a2f72 has session id: 48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829007406Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Upstairs opts: Upstairs UUID: 4314e392-46f2-4562-9f68-ef3de23a2f72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829052515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Crucible stats registered with UUID: 4314e392-46f2-4562-9f68-ef3de23a2f72","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829071874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:31.829084903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829474083Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829507352Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} {"msg":"4314e392-46f2-4562-9f68-ef3de23a2f72 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829575951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.82960298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"0","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829627819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"0","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829642949Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829657499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829670758Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"2","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829692167Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"2","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829720587Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829748816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829780415Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40133)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.829813194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:31.830127506Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49500)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.830174315Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:31.830206044Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:31.830243583Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.830275312Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.83073878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"2","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.830780699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"0","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.831420662Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"0","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.831549399Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"2","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.831914819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:31.832072945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.831712538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42004)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.831864534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.832858898Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.833493631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[0]R flush_numbers[0..12]: [286, 285, 281, 282, 280, 286, 282, 286, 286, 275, 281, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834169403Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[0]R generation[0..12]: [155, 154, 153, 154, 152, 155, 154, 155, 155, 149, 153, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834193863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834209593Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[1]R flush_numbers[0..12]: [286, 285, 281, 282, 280, 286, 282, 286, 286, 275, 281, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834228212Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[1]R generation[0..12]: [155, 154, 153, 154, 152, 155, 154, 155, 155, 149, 153, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834242481Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834256451Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[2]R flush_numbers[0..12]: [286, 285, 281, 282, 280, 286, 282, 286, 286, 275, 281, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834270341Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[2]R generation[0..12]: [155, 154, 153, 154, 152, 155, 154, 155, 155, 149, 153, 149]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834284691Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.83429885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.8343119Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834335109Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834349999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834363968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"":"downstairs","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834378588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834391058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"0","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834405257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"1","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834418887Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"client":"2","":"downstairs","session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834434856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834447926Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"4314e392-46f2-4562-9f68-ef3de23a2f72 is now active with session: 48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834461046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834473515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816,"session_id":"48a5e2cf-f206-46ce-bc7f-b0b5a6fac67f"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834486575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} {"Wait for a query_work_queue command to finish before sending IO msg":"Activated sub_volume 4314e392-46f2-4562-9f68-ef3de23a2f72","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:41.834504715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 351 len 16384 data: 11 10 7 10 02/30 Read block 199 len 24576 03/30 Write block 159 len 20480 data: 11 8 6 9 8 04/30 Write block 358 len 28672 data: 10 13 13 13 15 15 11 05/30 Read block 219 len 16384 06/30 Read block 491 len 40960 07/30 Write block 438 len 12288 data: 8 7 6 08/30 Read block 67 len 16384 09/30 Write block 321 len 28672 data: 13 10 12 11 11 12 10 10/30 Read block 383 len 4096 11/30 Write block 54 len 8192 data: 11 11 12/30 Write block 224 len 28672 data: 15 13 14 14 12 10 9 13/30 Write block 215 len 20480 data: 11 10 10 8 8 14/30 Read block 294 len 16384 15/30 Write block 457 len 12288 data: 11 12 11 16/30 Flush 17/30 Write block 308 len 28672 data: 17 19 16 15 14 15 17 18/30 Read block 326 len 8192 19/30 Write block 236 len 8192 data: 11 11 20/30 Write block 237 len 32768 data: 12 11 9 11 13 9 11 8 21/30 Read block 291 len 8192 22/30 Write block 433 len 8192 data: 9 9 23/30 Write block 588 len 40960 data: 9 8 10 9 9 8 8 7 9 7 24/30 Read block 151 len 20480 25/30 Read block 553 len 16384 26/30 Write block 4 len 28672 data: 7 8 11 9 7 11 12 27/30 Read block 52 len 8192 28/30 Write block 201 len 12288 data: 9 10 8 29/30 Read block 18 len 20480 30/30 Read block 91 len 16384 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 13 14 14 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:41.856087376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1816} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 286 286 286 T T T 1 020-039 154 154 154 285 285 285 F F F 2 040-059 156 156 156 287 287 287 F F F 3 060-079 154 154 154 282 282 282 F F F 4 080-099 152 152 152 280 280 280 F F F 5 100-119 155 155 155 286 286 286 F F F 6 120-139 154 154 154 282 282 282 F F F 7 140-159 156 156 156 287 287 287 F F F 8 160-179 156 156 156 287 287 287 F F F 9 180-199 149 149 149 275 275 275 F F F 10 200-219 156 156 156 287 287 287 T T T 11 220-239 156 156 156 287 287 287 T T T 12 240-259 153 153 153 281 281 281 T T T 13 260-279 155 155 155 286 286 286 F F F 14 280-299 155 155 155 286 286 286 F F F 15 300-319 155 155 155 286 286 286 T T T 16 320-339 156 156 156 287 287 287 F F F 17 340-359 156 156 156 287 287 287 F F F 18 360-379 156 156 156 287 287 287 F F F 19 380-399 155 155 155 286 286 286 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 156 156 156 287 287 287 T T T 22 440-459 156 156 156 287 287 287 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 151 151 151 278 278 278 F F F 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 151 151 151 278 278 278 T T T Max gen: 156, Max flush: 287 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 288 288 288 F F F 1 020-039 154 154 154 285 285 285 F F F 2 040-059 156 156 156 287 287 287 F F F 3 060-079 154 154 154 282 282 282 F F F 4 080-099 152 152 152 280 280 280 F F F 5 100-119 155 155 155 286 286 286 F F F 6 120-139 154 154 154 282 282 282 F F F 7 140-159 156 156 156 287 287 287 F F F 8 160-179 156 156 156 287 287 287 F F F 9 180-199 149 149 149 275 275 275 F F F 10 200-219 157 157 157 288 288 288 F F F 11 220-239 157 157 157 288 288 288 F F F 12 240-259 157 157 157 288 288 288 F F F 13 260-279 155 155 155 286 286 286 F F F 14 280-299 155 155 155 286 286 286 F F F 15 300-319 157 157 157 288 288 288 F F F 16 320-339 156 156 156 287 287 287 F F F 17 340-359 156 156 156 287 287 287 F F F 18 360-379 156 156 156 287 287 287 F F F 19 380-399 155 155 155 286 286 286 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 157 157 157 288 288 288 F F F 22 440-459 156 156 156 287 287 287 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 151 151 151 278 278 278 F F F 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 157 157 157 288 288 288 F F F Max gen: 157, Max flush: 288 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.10498264Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.158027903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.158754993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.158788723Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.158802342Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.158816412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} {"msg":"Crucible ba852fce-227d-4c8e-8900-8e0f1e42df6e has session id: ef677483-1333-483f-838c-ffafbbe28e31","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.159353538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Upstairs opts: Upstairs UUID: ba852fce-227d-4c8e-8900-8e0f1e42df6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.159435666Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Crucible stats registered with UUID: ba852fce-227d-4c8e-8900-8e0f1e42df6e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.159454575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:46.159468115Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.159911283Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.159937202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} {"msg":"ba852fce-227d-4c8e-8900-8e0f1e42df6e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.159977241Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.159992551Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"0","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.1600206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"0","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.16003749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"1","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160051689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"1","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160083158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"2","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160098098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"2","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160110838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160127837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160201305Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:33380)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160243684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:54935)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160472218Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:37378)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160521047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.160958295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"2","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.161101052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"0","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.161621978Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"2","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.161802483Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"0","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.162275601Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.162368948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.162730379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"1","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163261125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"1","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[0]R flush_numbers[0..12]: [288, 285, 287, 282, 280, 286, 282, 287, 287, 275, 288, 288]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163782701Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[0]R generation[0..12]: [157, 154, 156, 154, 152, 155, 154, 156, 156, 149, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.16380416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.16381921Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[1]R flush_numbers[0..12]: [288, 285, 287, 282, 280, 286, 282, 287, 287, 275, 288, 288]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.1638331Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[1]R generation[0..12]: [157, 154, 156, 154, 152, 155, 154, 156, 156, 149, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163846689Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163860709Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[2]R flush_numbers[0..12]: [288, 285, 287, 282, 280, 286, 282, 287, 287, 275, 288, 288]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163874529Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[2]R generation[0..12]: [157, 154, 156, 154, 152, 155, 154, 156, 156, 149, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163887728Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163901198Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163916737Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163929947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163942637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163955136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"":"downstairs","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163967876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163979776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"0","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.163992746Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"client":"1","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.164005565Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","Wait for a query_work_queue command to finish before sending IO pid":1824,"client":"2","":"downstairs","session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.164036104Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.164048654Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"ba852fce-227d-4c8e-8900-8e0f1e42df6e is now active with session: ef677483-1333-483f-838c-ffafbbe28e31","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.164075863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.164088793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824,"session_id":"ef677483-1333-483f-838c-ffafbbe28e31"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.164100912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} {"msg":"Activated sub_volume ba852fce-227d-4c8e-8900-8e0f1e42df6e","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:10:46.164111922Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 4096 data: 14 02/30 Write block 166 len 40960 data: 11 9 11 11 13 14 11 12 11 14 03/30 Read block 508 len 40960 04/30 Flush 05/30 Flush 06/30 Write block 17 len 32768 data: 14 15 14 12 11 11 10 10 07/30 Write block 360 len 28672 data: 14 14 16 16 12 10 13 08/30 Read block 251 len 24576 09/30 Write block 158 len 16384 data: 12 12 9 7 10/30 Write block 161 len 20480 data: 8 10 9 9 8 11/30 Read block 552 len 36864 12/30 Read block 55 len 40960 13/30 Flush 14/30 Write block 259 len 20480 data: 12 12 17 16 14 15/30 Write block 371 len 40960 data: 13 14 14 16 13 12 16 18 19 18 16/30 Flush 17/30 Write block 221 len 24576 data: 9 10 11 16 14 15 18/30 Write block 197 len 28672 data: 6 6 7 7 10 11 9 19/30 Write block 69 len 40960 data: 13 9 9 7 10 13 15 15 15 16 20/30 Read block 182 len 4096 21/30 Write block 487 len 12288 data: 14 12 11 22/30 Write block 213 len 20480 data: 12 11 12 11 11 23/30 Write block 438 len 36864 data: 9 8 7 6 7 6 7 9 10 24/30 Read block 380 len 8192 25/30 Write block 321 len 4096 data: 14 26/30 Write block 356 len 40960 data: 9 9 11 14 15 15 17 17 13 11 27/30 Write block 316 len 28672 data: 15 15 11 13 12 15 11 28/30 Read block 552 len 20480 29/30 Write block 167 len 20480 data: 10 12 12 14 15 30/30 Write block 482 len 40960 data: 12 11 11 11 15 15 13 12 10 9 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:10:46.18469526Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1824} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 291 291 291 F F F 1 020-039 158 158 158 291 291 291 F F F 2 040-059 156 156 156 287 287 287 F F F 3 060-079 154 154 154 282 282 282 T T T 4 080-099 158 158 158 289 289 289 F F F 5 100-119 155 155 155 286 286 286 F F F 6 120-139 154 154 154 282 282 282 F F F 7 140-159 158 158 158 291 291 291 F F F 8 160-179 158 158 158 291 291 291 T T T 9 180-199 149 149 149 275 275 275 T T T 10 200-219 157 157 157 288 288 288 T T T 11 220-239 157 157 157 288 288 288 T T T 12 240-259 158 158 158 292 292 292 F F F 13 260-279 158 158 158 292 292 292 F F F 14 280-299 155 155 155 286 286 286 F F F 15 300-319 157 157 157 288 288 288 T T T 16 320-339 156 156 156 287 287 287 T T T 17 340-359 156 156 156 287 287 287 T T T 18 360-379 158 158 158 292 292 292 T T T 19 380-399 158 158 158 292 292 292 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 157 157 157 288 288 288 T T T 22 440-459 156 156 156 287 287 287 T T T 23 460-479 154 154 154 283 283 283 F F F 24 480-499 151 151 151 278 278 278 T T T 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 157 157 157 288 288 288 F F F Max gen: 158, Max flush: 292 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 291 291 291 F F F 1 020-039 158 158 158 291 291 291 F F F 2 040-059 156 156 156 287 287 287 F F F 3 060-079 159 159 159 293 293 293 F F F 4 080-099 158 158 158 289 289 289 F F F 5 100-119 155 155 155 286 286 286 F F F 6 120-139 154 154 154 282 282 282 F F F 7 140-159 158 158 158 291 291 291 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 159 159 159 293 293 293 F F F 10 200-219 159 159 159 293 293 293 F F F 11 220-239 159 159 159 293 293 293 F F F 12 240-259 158 158 158 292 292 292 F F F 13 260-279 158 158 158 292 292 292 F F F 14 280-299 155 155 155 286 286 286 F F F 15 300-319 159 159 159 293 293 293 F F F 16 320-339 159 159 159 293 293 293 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 159 159 159 293 293 293 F F F 19 380-399 158 158 158 292 292 292 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 159 159 159 293 293 293 F F F 22 440-459 159 159 159 293 293 293 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 159 159 159 293 293 293 F F F 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 151 151 151 278 278 278 F F F 29 580-599 157 157 157 288 288 288 F F F Max gen: 159, Max flush: 293 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.456669867Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:00.498726379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.499110169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.529982446Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.53059532Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.530631549Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.530649808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.530664438Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} {"msg":"Crucible 5490e6e0-6e15-496f-9d19-b96a9369cee6 has session id: 3930867c-f8de-4eef-a917-2f1ec5f3f5b7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531284332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Upstairs opts: Upstairs UUID: 5490e6e0-6e15-496f-9d19-b96a9369cee6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.53132947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Crucible stats registered with UUID: 5490e6e0-6e15-496f-9d19-b96a9369cee6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:00.5313598Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.53173334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531755709Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} {"msg":"5490e6e0-6e15-496f-9d19-b96a9369cee6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531819548Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531836517Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"0","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531858067Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"0","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531871456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"1","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531885816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"1","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531898626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"2","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531911995Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"2","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531927645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.531981003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.532021982Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50415)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.532043102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58768)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.532069301Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45237)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.53209512Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.532746643Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"1","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.532777232Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"2","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.533187912Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"0","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.533662359Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"1","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.533707148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"2","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534030939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"0","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.53436573Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534443608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[0]R flush_numbers[0..12]: [291, 291, 287, 293, 289, 286, 282, 291, 293, 293, 293, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534563335Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[0]R generation[0..12]: [158, 158, 156, 159, 158, 155, 154, 158, 159, 159, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534584215Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534599044Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[1]R flush_numbers[0..12]: [291, 291, 287, 293, 289, 286, 282, 291, 293, 293, 293, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534612904Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[1]R generation[0..12]: [158, 158, 156, 159, 158, 155, 154, 158, 159, 159, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534626514Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534640593Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[2]R flush_numbers[0..12]: [291, 291, 287, 293, 289, 286, 282, 291, 293, 293, 293, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534654053Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[2]R generation[0..12]: [158, 158, 156, 159, 158, 155, 154, 158, 159, 159, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534667602Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534681332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534693932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534706901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534719471Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534732031Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"":"downstairs","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.5347448Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.53475673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"0","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.53477219Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"1","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534793789Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"client":"2","":"downstairs","session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534806829Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534818929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"5490e6e0-6e15-496f-9d19-b96a9369cee6 is now active with session: 3930867c-f8de-4eef-a917-2f1ec5f3f5b7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534831568Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534843228Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832,"session_id":"3930867c-f8de-4eef-a917-2f1ec5f3f5b7"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534855317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} {"msg":"Activated sub_volume 5490e6e0-6e15-496f-9d19-b96a9369cee6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:00.534866397Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 20480 data: 9 12 11 13 14 02/30 Read block 372 len 4096 03/30 Read block 343 len 4096 04/30 Read block 465 len 20480 05/30 Read block 89 len 36864 06/30 Read block 448 len 24576 07/30 Read block 557 len 12288 08/30 Write block 568 len 16384 data: 17 14 14 13 09/30 Read block 277 len 4096 10/30 Write block 431 len 28672 data: 13 9 10 10 8 10 9 11/30 Write block 278 len 12288 data: 13 12 9 12/30 Write block 18 len 20480 data: 16 15 13 12 12 13/30 Read block 473 len 40960 14/30 Write block 186 len 24576 data: 7 10 9 9 8 9 15/30 Read block 382 len 12288 16/30 Read block 185 len 4096 17/30 Write block 489 len 40960 data: 13 11 10 8 13 16 16 15 15 13 18/30 Read block 437 len 12288 19/30 Write block 566 len 36864 data: 18 16 18 15 15 14 14 14 13 20/30 Read block 281 len 20480 21/30 Read block 479 len 36864 22/30 Read block 433 len 36864 23/30 Read block 331 len 36864 24/30 Read block 454 len 28672 25/30 Write block 94 len 12288 data: 13 13 15 26/30 Flush 27/30 Write block 272 len 8192 data: 13 14 28/30 Read block 489 len 40960 29/30 Read block 590 len 4096 30/30 Write block 10 len 16384 data: 13 11 11 11 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:00.556172416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1832} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 294 294 294 T T T 1 020-039 160 160 160 294 294 294 F F F 2 040-059 160 160 160 294 294 294 F F F 3 060-079 160 160 160 294 294 294 F F F 4 080-099 160 160 160 294 294 294 F F F 5 100-119 155 155 155 286 286 286 F F F 6 120-139 154 154 154 282 282 282 F F F 7 140-159 158 158 158 291 291 291 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 160 160 160 294 294 294 F F F 10 200-219 159 159 159 293 293 293 F F F 11 220-239 159 159 159 293 293 293 F F F 12 240-259 158 158 158 292 292 292 F F F 13 260-279 160 160 160 294 294 294 T T T 14 280-299 160 160 160 294 294 294 F F F 15 300-319 159 159 159 293 293 293 F F F 16 320-339 159 159 159 293 293 293 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 159 159 159 293 293 293 F F F 19 380-399 158 158 158 292 292 292 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 160 160 160 294 294 294 F F F 22 440-459 159 159 159 293 293 293 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 160 160 160 294 294 294 F F F 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 160 160 160 294 294 294 F F F 29 580-599 157 157 157 288 288 288 F F F Max gen: 160, Max flush: 294 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 160 160 160 294 294 294 F F F 2 040-059 160 160 160 294 294 294 F F F 3 060-079 160 160 160 294 294 294 F F F 4 080-099 160 160 160 294 294 294 F F F 5 100-119 155 155 155 286 286 286 F F F 6 120-139 154 154 154 282 282 282 F F F 7 140-159 158 158 158 291 291 291 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 160 160 160 294 294 294 F F F 10 200-219 159 159 159 293 293 293 F F F 11 220-239 159 159 159 293 293 293 F F F 12 240-259 158 158 158 292 292 292 F F F 13 260-279 161 161 161 295 295 295 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 159 159 159 293 293 293 F F F 16 320-339 159 159 159 293 293 293 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 159 159 159 293 293 293 F F F 19 380-399 158 158 158 292 292 292 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 160 160 160 294 294 294 F F F 22 440-459 159 159 159 293 293 293 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 160 160 160 294 294 294 F F F 25 500-519 154 154 154 282 282 282 F F F 26 520-539 151 151 151 278 278 278 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 160 160 160 294 294 294 F F F 29 580-599 157 157 157 288 288 288 F F F Max gen: 161, Max flush: 295 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.804505465Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:04.847177251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.847660188Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.879256036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.880143033Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.880181812Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.880199631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.880213791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} {"msg":"Crucible 8ba36d90-c4e3-4be5-bdca-847712baca96 has session id: a627a3cf-818d-4694-9b6c-0c17ba7f0abd","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.880892563Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Upstairs opts: Upstairs UUID: 8ba36d90-c4e3-4be5-bdca-847712baca96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.880929892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Crucible stats registered with UUID: 8ba36d90-c4e3-4be5-bdca-847712baca96","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.880945121Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:04.880957601Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.88137183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.88139682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} {"msg":"8ba36d90-c4e3-4be5-bdca-847712baca96 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881457758Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881475347Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"0","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881497307Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"0","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881510756Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"1","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881524886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"1","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881538076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"2","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881551915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"2","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881564615Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881577975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881591934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35634)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881674622Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50592)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881719511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65245)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.881769979Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.882542249Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"1","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.882582168Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"2","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.882778243Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"0","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.883394397Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"1","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.883422916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"2","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.883438976Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"0","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.883983001Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884088329Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[0]R flush_numbers[0..12]: [295, 294, 294, 294, 294, 286, 282, 291, 293, 294, 293, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884156047Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[0]R generation[0..12]: [161, 160, 160, 160, 160, 155, 154, 158, 159, 160, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884175256Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884189626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[1]R flush_numbers[0..12]: [295, 294, 294, 294, 294, 286, 282, 291, 293, 294, 293, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884203475Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[1]R generation[0..12]: [161, 160, 160, 160, 160, 155, 154, 158, 159, 160, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884217165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884230905Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[2]R flush_numbers[0..12]: [295, 294, 294, 294, 294, 286, 282, 291, 293, 294, 293, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884244464Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[2]R generation[0..12]: [161, 160, 160, 160, 160, 155, 154, 158, 159, 160, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884261594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884275633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884288473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884302053Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884314552Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884327122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"":"downstairs","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884339872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884351542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"0","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-19T17:11:04.884364421Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"1","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884384261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"client":"2","":"downstairs","session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.8843971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.88441453Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"8ba36d90-c4e3-4be5-bdca-847712baca96 is now active with session: a627a3cf-818d-4694-9b6c-0c17ba7f0abd","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884427209Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884438829Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840,"session_id":"a627a3cf-818d-4694-9b6c-0c17ba7f0abd"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884450819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} {"msg":"Activated sub_volume 8ba36d90-c4e3-4be5-bdca-847712baca96","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:04.884461669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 20480 data: 16 17 20 21 19 02/30 Write block 205 len 36864 data: 9 11 10 9 10 11 12 14 13 03/30 Write block 247 len 20480 data: 15 16 16 16 16 04/30 Write block 211 len 28672 data: 13 15 14 12 13 12 12 05/30 Read block 282 len 28672 06/30 Read block 107 len 8192 07/30 Write block 41 len 28672 data: 11 11 12 13 13 10 11 08/30 Flush 09/30 Write block 292 len 32768 data: 6 9 11 10 10 8 10 9 10/30 Read block 166 len 16384 11/30 Read block 161 len 20480 12/30 Flush 13/30 Write block 74 len 12288 data: 14 16 16 14/30 Read block 352 len 12288 15/30 Read block 228 len 16384 16/30 Read block 355 len 12288 17/30 Write block 519 len 40960 data: 10 8 8 10 11 11 11 9 11 12 18/30 Write block 193 len 8192 data: 9 9 19/30 Write block 315 len 28672 data: 15 16 16 12 14 13 16 20/30 Write block 517 len 28672 data: 11 11 11 9 9 11 12 21/30 Write block 136 len 28672 data: 10 11 14 16 14 16 17 22/30 Write block 496 len 40960 data: 16 16 14 7 5 5 4 7 6 8 23/30 Write block 279 len 20480 data: 13 10 9 7 8 24/30 Write block 569 len 8192 data: 16 16 25/30 Read block 320 len 24576 26/30 Read block 406 len 4096 27/30 Read block 333 len 24576 28/30 Read block 232 len 36864 29/30 Read block 21 len 28672 30/30 Write block 264 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:04.904188149Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1840} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 160 160 160 294 294 294 F F F 2 040-059 162 162 162 296 296 296 F F F 3 060-079 160 160 160 294 294 294 T T T 4 080-099 162 162 162 296 296 296 F F F 5 100-119 162 162 162 296 296 296 F F F 6 120-139 154 154 154 282 282 282 T T T 7 140-159 158 158 158 291 291 291 T T T 8 160-179 159 159 159 293 293 293 F F F 9 180-199 160 160 160 294 294 294 T T T 10 200-219 162 162 162 296 296 296 F F F 11 220-239 159 159 159 293 293 293 F F F 12 240-259 162 162 162 296 296 296 F F F 13 260-279 161 161 161 295 295 295 T T T 14 280-299 162 162 162 297 297 297 T T T 15 300-319 159 159 159 293 293 293 T T T 16 320-339 159 159 159 293 293 293 T T T 17 340-359 159 159 159 293 293 293 F F F 18 360-379 159 159 159 293 293 293 F F F 19 380-399 158 158 158 292 292 292 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 160 160 160 294 294 294 F F F 22 440-459 159 159 159 293 293 293 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 160 160 160 294 294 294 T T T 25 500-519 154 154 154 282 282 282 T T T 26 520-539 151 151 151 278 278 278 T T T 27 540-559 155 155 155 286 286 286 F F F 28 560-579 160 160 160 294 294 294 T T T 29 580-599 157 157 157 288 288 288 F F F Max gen: 162, Max flush: 297 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 160 160 160 294 294 294 F F F 2 040-059 162 162 162 296 296 296 F F F 3 060-079 163 163 163 298 298 298 F F F 4 080-099 162 162 162 296 296 296 F F F 5 100-119 162 162 162 296 296 296 F F F 6 120-139 163 163 163 298 298 298 F F F 7 140-159 163 163 163 298 298 298 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 162 162 162 296 296 296 F F F 11 220-239 159 159 159 293 293 293 F F F 12 240-259 162 162 162 296 296 296 F F F 13 260-279 163 163 163 298 298 298 F F F 14 280-299 163 163 163 298 298 298 F F F 15 300-319 163 163 163 298 298 298 F F F 16 320-339 163 163 163 298 298 298 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 159 159 159 293 293 293 F F F 19 380-399 158 158 158 292 292 292 F F F 20 400-419 155 155 155 286 286 286 F F F 21 420-439 160 160 160 294 294 294 F F F 22 440-459 159 159 159 293 293 293 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 163 163 163 298 298 298 F F F 25 500-519 163 163 163 298 298 298 F F F 26 520-539 163 163 163 298 298 298 F F F 27 540-559 155 155 155 286 286 286 F F F 28 560-579 163 163 163 298 298 298 F F F 29 580-599 157 157 157 288 288 288 F F F Max gen: 163, Max flush: 298 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.207241627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.250657133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.251444782Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.251479182Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.251496971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.251511671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} {"msg":"Crucible 556ab8e8-0f78-490b-a234-518ae640f393 has session id: 24e0352f-6f08-47b8-9e09-0c72cbb80f29","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252004808Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Upstairs opts: Upstairs UUID: 556ab8e8-0f78-490b-a234-518ae640f393, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252055837Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Crucible stats registered with UUID: 556ab8e8-0f78-490b-a234-518ae640f393","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252074626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:09.252087776Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252660841Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.25269368Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} {"msg":"556ab8e8-0f78-490b-a234-518ae640f393 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252749518Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252777247Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"0","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252813817Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"0","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252840906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"1","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252869395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"1","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252898234Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252929594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252948993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.252974372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.253000412Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:09.253028961Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:09.253061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:09.253088609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"ds_connection connected from Ok(127.0.0.1:37228)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.253131348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.253163677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"ds_connection connected from Ok(127.0.0.1:61303)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.253304024Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.253726853Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"0","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.253847859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"1","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.254237659Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"0","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.254350836Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"1","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.254857663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:09.25495182Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.253557649Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"ds_connection connected from Ok(127.0.0.1:63495)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.253627807Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.254728108Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.255380731Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[0]R flush_numbers[0..12]: [295, 294, 296, 298, 296, 296, 298, 298, 293, 298, 296, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256031814Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[0]R generation[0..12]: [161, 160, 162, 163, 162, 162, 163, 163, 159, 163, 162, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256060783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256077443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[1]R flush_numbers[0..12]: [295, 294, 296, 298, 296, 296, 298, 298, 293, 298, 296, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256091652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[1]R generation[0..12]: [161, 160, 162, 163, 162, 162, 163, 163, 159, 163, 162, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256105372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256119122Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[2]R flush_numbers[0..12]: [295, 294, 296, 298, 296, 296, 298, 298, 293, 298, 296, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256132571Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[2]R generation[0..12]: [161, 160, 162, 163, 162, 162, 163, 163, 159, 163, 162, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256145671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256163501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.25617635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.25618915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.25620166Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:11:19.256214119Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"":"downstairs","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256229749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256241788Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"0","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256254818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"1","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256267498Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"client":"2","":"downstairs","session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256280367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256292127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"556ab8e8-0f78-490b-a234-518ae640f393 is now active with session: 24e0352f-6f08-47b8-9e09-0c72cbb80f29","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256304717Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256316477Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848,"session_id":"24e0352f-6f08-47b8-9e09-0c72cbb80f29"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256328446Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} {"msg":"Activated sub_volume 556ab8e8-0f78-490b-a234-518ae640f393","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:19.256339306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 143 len 20480 data: 18 16 17 17 16 02/30 Write block 71 len 16384 data: 10 8 11 15 03/30 Read block 336 len 12288 04/30 Write block 318 len 36864 data: 13 15 14 17 12 13 12 12 13 05/30 Write block 508 len 28672 data: 12 11 12 15 13 13 15 06/30 Read block 68 len 36864 07/30 Write block 585 len 24576 data: 11 11 13 10 9 11 08/30 Flush 09/30 Write block 364 len 32768 data: 14 12 14 15 14 11 14 14 10/30 Read block 66 len 12288 11/30 Read block 220 len 12288 12/30 Write block 265 len 24576 data: 11 14 16 16 16 17 13/30 Write block 44 len 20480 data: 14 14 11 12 9 14/30 Read block 371 len 4096 15/30 Write block 55 len 12288 data: 12 10 13 16/30 Write block 324 len 8192 data: 13 13 17/30 Write block 84 len 28672 data: 14 11 13 15 15 17 16 18/30 Write block 555 len 36864 data: 13 10 8 7 7 9 10 11 13 19/30 Read block 482 len 4096 20/30 Write block 520 len 40960 data: 10 10 12 13 12 12 10 12 13 11 21/30 Read block 425 len 36864 22/30 Read block 163 len 16384 23/30 Read block 191 len 24576 24/30 Flush 25/30 Write block 113 len 32768 data: 12 16 16 15 16 14 13 12 26/30 Read block 35 len 8192 27/30 Read block 299 len 32768 28/30 Read block 232 len 8192 29/30 Write block 399 len 8192 data: 21 22 30/30 Write block 16 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:19.277156847Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1848} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 T T T 1 020-039 160 160 160 294 294 294 F F F 2 040-059 164 164 164 300 300 300 F F F 3 060-079 164 164 164 299 299 299 F F F 4 080-099 164 164 164 300 300 300 F F F 5 100-119 162 162 162 296 296 296 T T T 6 120-139 163 163 163 298 298 298 T T T 7 140-159 164 164 164 299 299 299 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 162 162 162 296 296 296 F F F 11 220-239 159 159 159 293 293 293 F F F 12 240-259 162 162 162 296 296 296 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 163 163 163 298 298 298 F F F 15 300-319 164 164 164 299 299 299 F F F 16 320-339 164 164 164 300 300 300 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 158 158 158 292 292 292 T T T 20 400-419 155 155 155 286 286 286 T T T 21 420-439 160 160 160 294 294 294 F F F 22 440-459 159 159 159 293 293 293 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 163 163 163 298 298 298 F F F 25 500-519 164 164 164 299 299 299 F F F 26 520-539 164 164 164 300 300 300 F F F 27 540-559 164 164 164 300 300 300 F F F 28 560-579 164 164 164 300 300 300 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 164, Max flush: 300 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 301 301 301 F F F 1 020-039 160 160 160 294 294 294 F F F 2 040-059 164 164 164 300 300 300 F F F 3 060-079 164 164 164 299 299 299 F F F 4 080-099 164 164 164 300 300 300 F F F 5 100-119 165 165 165 301 301 301 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 164 164 164 299 299 299 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 162 162 162 296 296 296 F F F 11 220-239 159 159 159 293 293 293 F F F 12 240-259 162 162 162 296 296 296 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 163 163 163 298 298 298 F F F 15 300-319 164 164 164 299 299 299 F F F 16 320-339 164 164 164 300 300 300 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 165 165 165 301 301 301 F F F 20 400-419 165 165 165 301 301 301 F F F 21 420-439 160 160 160 294 294 294 F F F 22 440-459 159 159 159 293 293 293 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 163 163 163 298 298 298 F F F 25 500-519 164 164 164 299 299 299 F F F 26 520-539 164 164 164 300 300 300 F F F 27 540-559 164 164 164 300 300 300 F F F 28 560-579 164 164 164 300 300 300 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 165, Max flush: 301 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.517095587Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:23.559441652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.559942899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.590892883Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.591550346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.591582945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.591595965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.591614304Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} {"msg":"Crucible 8bfcacef-f131-4066-b00f-05a66a1886d9 has session id: cb4bb753-3db3-4966-8f9b-deb6643e1ac7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592264087Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Upstairs opts: Upstairs UUID: 8bfcacef-f131-4066-b00f-05a66a1886d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592295666Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Crucible stats registered with UUID: 8bfcacef-f131-4066-b00f-05a66a1886d9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592309856Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:23.592321895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592802663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592824612Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} {"msg":"8bfcacef-f131-4066-b00f-05a66a1886d9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592880411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.59290633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"0","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592930019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"0","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592943879Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"1","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592957849Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"1","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592971778Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"2","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.592985468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"2","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593031687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593069536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593095925Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36954)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593120005Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"ds_connection connected from Ok(127.0.0.1:42064)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593142294Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37972)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593165293Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593854805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"1","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.593890654Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"2","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.59406277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"0","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.594492578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"1","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.594524577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"2","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.594648924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"0","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595094562Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595126521Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[0]R flush_numbers[0..12]: [301, 294, 300, 299, 300, 301, 301, 299, 293, 298, 296, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595255098Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[0]R generation[0..12]: [165, 160, 164, 164, 164, 165, 165, 164, 159, 163, 162, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595278688Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595293827Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[1]R flush_numbers[0..12]: [301, 294, 300, 299, 300, 301, 301, 299, 293, 298, 296, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595307397Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[1]R generation[0..12]: [165, 160, 164, 164, 164, 165, 165, 164, 159, 163, 162, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595320987Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595334696Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[2]R flush_numbers[0..12]: [301, 294, 300, 299, 300, 301, 301, 299, 293, 298, 296, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595348206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[2]R generation[0..12]: [165, 160, 164, 164, 164, 165, 165, 164, 159, 163, 162, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595361495Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595375075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595387645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595400504Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595412924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595425434Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6Wait for a query_work_queue command to finish before sending IO ","pid":1856,"":"downstairs","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595445383Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595456983Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"0","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595471643Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"1","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595484802Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"client":"2","":"downstairs","session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595497852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595509631Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"8bfcacef-f131-4066-b00f-05a66a1886d9 is now active with session: cb4bb753-3db3-4966-8f9b-deb6643e1ac7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595522231Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595533821Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856,"session_id":"cb4bb753-3db3-4966-8f9b-deb6643e1ac7"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.595545581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} {"msg":"Activated sub_volume 8bfcacef-f131-4066-b00f-05a66a1886d9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:23.5955566Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 530 len 32768 data: 9 9 11 12 12 15 16 16 02/30 Read block 46 len 36864 03/30 Flush 04/30 Write block 35 len 16384 data: 14 13 11 8 05/30 Write block 224 len 16384 data: 17 15 16 15 06/30 Read block 457 len 24576 07/30 Read block 435 len 24576 08/30 Read block 517 len 36864 09/30 Read block 114 len 24576 10/30 Write block 529 len 8192 data: 12 10 11/30 Read block 145 len 36864 12/30 Flush 13/30 Write block 288 len 12288 data: 3 6 8 14/30 Read block 458 len 36864 15/30 Read block 151 len 16384 16/30 Write block 498 len 24576 data: 15 8 6 6 5 8 17/30 Read block 267 len 4096 18/30 Read block 413 len 32768 19/30 Read block 559 len 8192 20/30 Write block 151 len 16384 data: 7 8 7 7 21/30 Write block 457 len 8192 data: 12 13 22/30 Read block 361 len 28672 23/30 Read block 426 len 8192 24/30 Read block 285 len 32768 25/30 Write block 439 len 8192 data: 9 8 26/30 Read block 326 len 24576 27/30 Read block 102 len 28672 28/30 Write block 249 len 20480 data: 17 17 17 16 15 29/30 Write block 13 len 4096 data: 12 30/30 Read block 118 len 16384 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 16 18 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:23.61568311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 301 301 301 T T T 1 020-039 166 166 166 303 303 303 F F F 2 040-059 164 164 164 300 300 300 F F F 3 060-079 164 164 164 299 299 299 F F F 4 080-099 164 164 164 300 300 300 F F F 5 100-119 165 165 165 301 301 301 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 164 164 164 299 299 299 T T T 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 162 162 162 296 296 296 F F F 11 220-239 166 166 166 303 303 303 F F F 12 240-259 162 162 162 296 296 296 T T T 13 260-279 164 164 164 300 300 300 F F F 14 280-299 163 163 163 298 298 298 T T T 15 300-319 164 164 164 299 299 299 F F F 16 320-339 164 164 164 300 300 300 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 165 165 165 301 301 301 F F F 20 400-419 165 165 165 301 301 301 F F F 21 420-439 160 160 160 294 294 294 T T T 22 440-459 159 159 159 293 293 293 T T T 23 460-479 154 154 154 283 283 283 F F F 24 480-499 163 163 163 298 298 298 T T T 25 500-519 164 164 164 299 299 299 T T T 26 520-539 166 166 166 303 303 303 F F F 27 540-559 164 164 164 300 300 300 F F F 28 560-579 164 164 164 300 300 300 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 166, Max flush: 303 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 F F F 1 020-039 166 166 166 303 303 303 F F F 2 040-059 164 164 164 300 300 300 F F F 3 060-079 164 164 164 299 299 299 F F F 4 080-099 164 164 164 300 300 300 F F F 5 100-119 165 165 165 301 301 301 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 167 167 167 304 304 304 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 162 162 162 296 296 296 F F F 11 220-239 166 166 166 303 303 303 F F F 12 240-259 167 167 167 304 304 304 F F F 13 260-279 164 164 164 300 300 300 F F F 14 280-299 167 167 167 304 304 304 F F F 15 300-319 164 164 164 299 299 299 F F F 16 320-339 164 164 164 300 300 300 F F F 17 340-359 159 159 159 293 293 293 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 165 165 165 301 301 301 F F F 20 400-419 165 165 165 301 301 301 F F F 21 420-439 167 167 167 304 304 304 F F F 22 440-459 167 167 167 304 304 304 F F F 23 460-479 154 154 154 283 283 283 F F F 24 480-499 167 167 167 304 304 304 F F F 25 500-519 167 167 167 304 304 304 F F F 26 520-539 166 166 166 303 303 303 F F F 27 540-559 164 164 164 300 300 300 F F F 28 560-579 164 164 164 300 300 300 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 167, Max flush: 304 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.90037877Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.943470145Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.944116608Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.944151357Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.944165517Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.944179756Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} {"msg":"Crucible 7b4b56a0-fa0f-4baf-b62d-74e1da6e3a7d has session id: 943e2b8d-c3b5-4b80-8295-66673882d8b3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.944649714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Upstairs opts: Upstairs UUID: 7b4b56a0-fa0f-4baf-b62d-74e1da6e3a7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.944708012Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Crucible stats registered with UUID: 7b4b56a0-fa0f-4baf-b62d-74e1da6e3a7d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.944730102Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:27.944758691Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945304756Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945333146Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} {"msg":"7b4b56a0-fa0f-4baf-b62d-74e1da6e3a7d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945393894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945418584Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"0","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945442603Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"0","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945456432Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"1","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945480032Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"1","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945505301Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"2","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.94553321Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"2","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.94556037Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945586549Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945614128Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:61498)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945642578Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48869)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945670097Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:50078)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.945697196Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.946270351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"0","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.94630366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"2","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.946975352Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"0","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.947011262Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"2","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.947555967Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.947631125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.947856699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"1","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.948393915Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"1","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[0]R flush_numbers[0..12]: [304, 303, 300, 299, 300, 301, 301, 304, 293, 298, 296, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.948896442Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[0]R generation[0..12]: [167, 166, 164, 164, 164, 165, 165, 167, 159, 163, 162, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.948917071Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.948931671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[1]R flush_numbers[0..12]: [304, 303, 300, 299, 300, 301, 301, 304, 293, 298, 296, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.948945211Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[1]R generation[0..12]: [167, 166, 164, 164, 164, 165, 165, 167, 159, 163, 162, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.94895847Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.94897216Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[2]R flush_numbers[0..12]: [304, 303, 300, 299, 300, 301, 301, 304, 293, 298, 296, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.94898535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[2]R generation[0..12]: [167, 166, 164, 164, 164, 165, 165, 167, 159, 163, 162, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949001809Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949015799Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949028658Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949041658Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949054308Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3Wait for a query_work_queue command to finish before sending IO "} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949073477Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"":"downstairs","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949086337Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949097917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"0","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949110756Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"1","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949123426Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"client":"2","":"downstairs","session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949136156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949147895Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"7b4b56a0-fa0f-4baf-b62d-74e1da6e3a7d is now active with session: 943e2b8d-c3b5-4b80-8295-66673882d8b3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949160165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949171825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864,"session_id":"943e2b8d-c3b5-4b80-8295-66673882d8b3"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949183764Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} {"msg":"Activated sub_volume 7b4b56a0-fa0f-4baf-b62d-74e1da6e3a7d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:27.949194574Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 74 len 24576 data: 16 17 17 16 17 13 02/30 Write block 32 len 36864 data: 16 15 13 15 14 12 9 8 10 03/30 Read block 65 len 20480 04/30 Read block 171 len 12288 05/30 Read block 407 len 32768 06/30 Flush 07/30 Write block 547 len 36864 data: 19 19 17 17 17 13 15 12 14 08/30 Write block 401 len 32768 data: 19 16 15 13 13 12 14 15 09/30 Write block 452 len 16384 data: 14 13 14 13 10/30 Read block 306 len 28672 11/30 Read block 137 len 40960 12/30 Write block 499 len 28672 data: 9 7 7 6 9 7 9 13/30 Read block 452 len 24576 14/30 Write block 459 len 20480 data: 12 9 9 9 9 15/30 Write block 539 len 4096 data: 16 16/30 Flush 17/30 Write block 557 len 16384 data: 9 8 8 10 18/30 Read block 251 len 36864 19/30 Write block 351 len 8192 data: 12 11 20/30 Flush 21/30 Write block 274 len 36864 data: 16 18 16 15 14 14 11 10 8 22/30 Write block 68 len 4096 data: 15 23/30 Read block 520 len 12288 24/30 Flush 25/30 Write block 506 len 40960 data: 7 8 13 12 13 16 14 14 16 12 26/30 Flush 27/30 Write block 4 len 24576 data: 8 9 12 10 8 12 28/30 Write block 299 len 16384 data: 10 10 10 11 29/30 Read block 578 len 32768 30/30 Write block 151 len 36864 data: 8 9 8 8 10 11 13 13 13 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:27.97022429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1864} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 304 304 304 T T T 1 020-039 168 168 168 305 305 305 F F F 2 040-059 168 168 168 305 305 305 F F F 3 060-079 168 168 168 308 308 308 F F F 4 080-099 164 164 164 300 300 300 F F F 5 100-119 165 165 165 301 301 301 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 167 167 167 304 304 304 T T T 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 162 162 162 296 296 296 F F F 11 220-239 166 166 166 303 303 303 F F F 12 240-259 167 167 167 304 304 304 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 168 168 168 308 308 308 T T T 15 300-319 164 164 164 299 299 299 T T T 16 320-339 164 164 164 300 300 300 F F F 17 340-359 168 168 168 307 307 307 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 165 165 165 301 301 301 F F F 20 400-419 168 168 168 306 306 306 F F F 21 420-439 167 167 167 304 304 304 F F F 22 440-459 168 168 168 306 306 306 F F F 23 460-479 168 168 168 306 306 306 F F F 24 480-499 168 168 168 306 306 306 F F F 25 500-519 168 168 168 309 309 309 F F F 26 520-539 168 168 168 306 306 306 F F F 27 540-559 168 168 168 307 307 307 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 168, Max flush: 309 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 168 168 168 305 305 305 F F F 2 040-059 168 168 168 305 305 305 F F F 3 060-079 168 168 168 308 308 308 F F F 4 080-099 164 164 164 300 300 300 F F F 5 100-119 165 165 165 301 301 301 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 169 169 169 310 310 310 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 162 162 162 296 296 296 F F F 11 220-239 166 166 166 303 303 303 F F F 12 240-259 167 167 167 304 304 304 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 169 169 169 310 310 310 F F F 15 300-319 169 169 169 310 310 310 F F F 16 320-339 164 164 164 300 300 300 F F F 17 340-359 168 168 168 307 307 307 F F F 18 360-379 164 164 164 300 300 300 F F F 19 380-399 165 165 165 301 301 301 F F F 20 400-419 168 168 168 306 306 306 F F F 21 420-439 167 167 167 304 304 304 F F F 22 440-459 168 168 168 306 306 306 F F F 23 460-479 168 168 168 306 306 306 F F F 24 480-499 168 168 168 306 306 306 F F F 25 500-519 168 168 168 309 309 309 F F F 26 520-539 168 168 168 306 306 306 F F F 27 540-559 168 168 168 307 307 307 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 169, Max flush: 310 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.206100167Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:32.248345084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.248763953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.280051549Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.28075688Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.280791239Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.280809499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.280824298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} {"msg":"Crucible 2cc429de-dbd5-43ed-8b45-025753a7810d has session id: 3d61ba53-7fa3-42b3-a029-2b6e08f9fba6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.281441722Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Upstairs opts: Upstairs UUID: 2cc429de-dbd5-43ed-8b45-025753a7810d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.281490141Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Crucible stats registered with UUID: 2cc429de-dbd5-43ed-8b45-025753a7810d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.281513751Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:32.28152751Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.281945939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.281971748Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} {"msg":"2cc429de-dbd5-43ed-8b45-025753a7810d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282015207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282029607Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"0","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282051486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"0","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282065136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"1","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282079685Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"1","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282092945Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"2","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282106675Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"2","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282126464Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282153874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282180133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60229)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282247461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41608)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282459626Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46415)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.282524414Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.28304077Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"1","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.28306933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"2","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.283283644Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"0","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.283668384Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"1","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.283695013Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"2","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.283752052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"0","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284337816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284363745Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[0]R flush_numbers[0..12]: [310, 305, 305, 308, 300, 301, 301, 310, 293, 298, 296, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284390125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[0]R generation[0..12]: [169, 168, 168, 168, 164, 165, 165, 169, 159, 163, 162, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284409474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284424034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[1]R flush_numbers[0..12]: [310, 305, 305, 308, 300, 301, 301, 310, 293, 298, 296, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284437533Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[1]R generation[0..12]: [169, 168, 168, 168, 164, 165, 165, 169, 159, 163, 162, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284450953Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284464393Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[2]R flush_numbers[0..12]: [310, 305, 305, 308, 300, 301, 301, 310, 293, 298, 296, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284477692Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[2]R generation[0..12]: [169, 168, 168, 168, 164, 165, 165, 169, 159, 163, 162, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284490902Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284504252Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284516781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284529661Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284541991Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.28455449Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"":"downstairs","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.28456734Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.28457905Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"Wait for a query_work_queue command to finish before sending IO client":"0","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284601519Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"1","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284614489Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"client":"2","":"downstairs","session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284627548Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284639438Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"2cc429de-dbd5-43ed-8b45-025753a7810d is now active with session: 3d61ba53-7fa3-42b3-a029-2b6e08f9fba6","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284651908Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284663348Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872,"session_id":"3d61ba53-7fa3-42b3-a029-2b6e08f9fba6"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284675117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} {"msg":"Activated sub_volume 2cc429de-dbd5-43ed-8b45-025753a7810d","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:32.284686147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 435 len 16384 data: 9 11 10 10 02/30 Write block 52 len 4096 data: 9 03/30 Write block 325 len 24576 data: 14 14 11 11 11 11 04/30 Read block 165 len 20480 05/30 Flush 06/30 Flush 07/30 Write block 83 len 24576 data: 16 15 12 14 16 16 08/30 Read block 260 len 20480 09/30 Read block 279 len 28672 10/30 Flush 11/30 Write block 103 len 8192 data: 19 18 12/30 Write block 553 len 24576 data: 16 13 15 11 10 9 13/30 Write block 207 len 20480 data: 11 10 11 12 14 14/30 Flush 15/30 Flush 16/30 Flush 17/30 Write block 305 len 32768 data: 15 15 15 18 20 17 16 15 18/30 Write block 353 len 28672 data: 8 11 10 10 10 12 15 19/30 Write block 538 len 28672 data: 16 17 13 13 15 17 16 20/30 Read block 56 len 8192 21/30 Write block 503 len 4096 data: 10 22/30 Write block 448 len 12288 data: 7 6 11 23/30 Read block 83 len 8192 24/30 Write block 339 len 32768 data: 16 18 18 18 17 19 17 16 25/30 Read block 505 len 40960 26/30 Flush 27/30 Write block 374 len 32768 data: 17 14 13 17 19 20 19 15 28/30 Read block 329 len 16384 29/30 Write block 463 len 32768 data: 10 9 8 9 8 6 8 10 30/30 Read block 374 len 40960 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:32.304786807Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1872} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 168 168 168 305 305 305 F F F 2 040-059 170 170 170 311 311 311 F F F 3 060-079 168 168 168 308 308 308 F F F 4 080-099 170 170 170 313 313 313 F F F 5 100-119 170 170 170 314 314 314 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 169 169 169 310 310 310 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 170 170 170 314 314 314 F F F 11 220-239 166 166 166 303 303 303 F F F 12 240-259 167 167 167 304 304 304 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 169 169 169 310 310 310 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 170 170 170 317 317 317 F F F 17 340-359 170 170 170 317 317 317 F F F 18 360-379 164 164 164 300 300 300 T T T 19 380-399 165 165 165 301 301 301 T T T 20 400-419 168 168 168 306 306 306 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 170 170 170 317 317 317 F F F 23 460-479 168 168 168 306 306 306 T T T 24 480-499 168 168 168 306 306 306 F F F 25 500-519 170 170 170 317 317 317 F F F 26 520-539 170 170 170 317 317 317 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 170, Max flush: 317 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 168 168 168 305 305 305 F F F 2 040-059 170 170 170 311 311 311 F F F 3 060-079 168 168 168 308 308 308 F F F 4 080-099 170 170 170 313 313 313 F F F 5 100-119 170 170 170 314 314 314 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 169 169 169 310 310 310 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 170 170 170 314 314 314 F F F 11 220-239 166 166 166 303 303 303 F F F 12 240-259 167 167 167 304 304 304 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 169 169 169 310 310 310 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 170 170 170 317 317 317 F F F 17 340-359 170 170 170 317 317 317 F F F 18 360-379 171 171 171 318 318 318 F F F 19 380-399 171 171 171 318 318 318 F F F 20 400-419 168 168 168 306 306 306 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 170 170 170 317 317 317 F F F 23 460-479 171 171 171 318 318 318 F F F 24 480-499 168 168 168 306 306 306 F F F 25 500-519 170 170 170 317 317 317 F F F 26 520-539 170 170 170 317 317 317 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 171, Max flush: 318 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.565286466Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.609090552Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.609752594Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.609787993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.609801933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.609816343Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} {"msg":"Crucible 038a9617-1e3c-474f-84b6-e3c8f76e7bb3 has session id: 7c45a68f-ace4-4a77-991e-1df77b98f122","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.610432416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Upstairs opts: Upstairs UUID: 038a9617-1e3c-474f-84b6-e3c8f76e7bb3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.610489315Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Crucible stats registered with UUID: 038a9617-1e3c-474f-84b6-e3c8f76e7bb3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.610515834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:36.610537294Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.610901894Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.610925823Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} {"msg":"038a9617-1e3c-474f-84b6-e3c8f76e7bb3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.610992892Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611009971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"0","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611037111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"0","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.61105261Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.61106738Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611080969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"2","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611100479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"2","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611125058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611150617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611174637Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"ds_connection connected from Ok(127.0.0.1:54343)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611198526Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"ds_connection connected from Ok(127.0.0.1:38031)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611268194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:36.611470629Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:36.611518588Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:36.611538597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.611675114Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.612115512Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"2","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.612244489Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"0","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.612718216Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"2","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.612848313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"0","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.613251712Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:36.613429228Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.613306354Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"ds_connection connected from Ok(127.0.0.1:53713)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.613384442Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.614346876Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.614932431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[0]R flush_numbers[0..12]: [310, 305, 311, 308, 313, 314, 301, 310, 293, 298, 314, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615510006Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[0]R generation[0..12]: [169, 168, 170, 168, 170, 170, 165, 169, 159, 163, 170, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615538885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615555085Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[1]R flush_numbers[0..12]: [310, 305, 311, 308, 313, 314, 301, 310, 293, 298, 314, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615569264Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[1]R generation[0..12]: [169, 168, 170, 168, 170, 170, 165, 169, 159, 163, 170, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615587933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615601923Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[2]R flush_numbers[0..12]: [310, 305, 311, 308, 313, 314, 301, 310, 293, 298, 314, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615615313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[2]R generation[0..12]: [169, 168, 170, 168, 170, 170, 165, 169, 159, 163, 170, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615628683Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615642282Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615654822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615667671Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615680301Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615692921Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"":"downstairs","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615705901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.61571742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"0","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.61573032Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"1","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615743029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"client":"2","":"downstairs","session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615755819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615767649Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"038a9617-1e3c-474f-84b6-e3c8f76e7bb3 is now active with session: 7c45a68f-ace4-4a77-991e-1df77b98f122","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:11:46.615781408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615798078Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880,"session_id":"7c45a68f-ace4-4a77-991e-1df77b98f122"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615810068Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} {"msg":"Activated sub_volume 038a9617-1e3c-474f-84b6-e3c8f76e7bb3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:46.615820827Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 478 len 36864 data: 11 13 13 13 13 12 12 12 16 02/30 Read block 485 len 40960 03/30 Flush 04/30 Flush 05/30 Write block 402 len 28672 data: 17 16 14 14 13 15 16 06/30 Flush 07/30 Write block 405 len 24576 data: 15 14 16 17 14 15 08/30 Read block 574 len 16384 09/30 Flush 10/30 Write block 62 len 8192 data: 14 15 11/30 Read block 549 len 32768 12/30 Write block 145 len 4096 data: 18 13/30 Flush 14/30 Write block 233 len 36864 data: 6 6 11 12 13 12 10 12 14 15/30 Read block 490 len 32768 16/30 Read block 243 len 40960 17/30 Write block 281 len 28672 data: 11 9 9 8 7 6 5 18/30 Write block 69 len 40960 data: 14 10 11 9 12 17 18 18 17 18 19/30 Read block 401 len 8192 20/30 Read block 82 len 28672 21/30 Read block 423 len 8192 22/30 Write block 536 len 4096 data: 17 23/30 Write block 209 len 16384 data: 12 13 15 16 24/30 Read block 324 len 8192 25/30 Read block 339 len 28672 26/30 Flush 27/30 Write block 378 len 40960 data: 20 21 20 16 10 9 8 7 8 8 28/30 Read block 38 len 40960 29/30 Write block 54 len 16384 data: 12 13 11 14 30/30 Read block 160 len 24576 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:46.637470007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1880} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 168 168 168 305 305 305 F F F 2 040-059 170 170 170 311 311 311 T T T 3 060-079 172 172 172 324 324 324 F F F 4 080-099 170 170 170 313 313 313 F F F 5 100-119 170 170 170 314 314 314 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 172 172 172 323 323 323 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 172 172 172 324 324 324 F F F 11 220-239 172 172 172 324 324 324 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 172 172 172 324 324 324 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 170 170 170 317 317 317 F F F 17 340-359 170 170 170 317 317 317 F F F 18 360-379 171 171 171 318 318 318 T T T 19 380-399 171 171 171 318 318 318 T T T 20 400-419 172 172 172 322 322 322 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 170 170 170 317 317 317 F F F 23 460-479 172 172 172 319 319 319 F F F 24 480-499 172 172 172 319 319 319 F F F 25 500-519 170 170 170 317 317 317 F F F 26 520-539 172 172 172 324 324 324 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 172, Max flush: 324 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 168 168 168 305 305 305 F F F 2 040-059 173 173 173 325 325 325 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 170 170 170 313 313 313 F F F 5 100-119 170 170 170 314 314 314 F F F 6 120-139 165 165 165 301 301 301 F F F 7 140-159 172 172 172 323 323 323 F F F 8 160-179 159 159 159 293 293 293 F F F 9 180-199 163 163 163 298 298 298 F F F 10 200-219 172 172 172 324 324 324 F F F 11 220-239 172 172 172 324 324 324 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 172 172 172 324 324 324 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 170 170 170 317 317 317 F F F 17 340-359 170 170 170 317 317 317 F F F 18 360-379 173 173 173 325 325 325 F F F 19 380-399 173 173 173 325 325 325 F F F 20 400-419 172 172 172 322 322 322 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 170 170 170 317 317 317 F F F 23 460-479 172 172 172 319 319 319 F F F 24 480-499 172 172 172 319 319 319 F F F 25 500-519 170 170 170 317 317 317 F F F 26 520-539 172 172 172 324 324 324 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 173, Max flush: 325 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.865695605Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:50.908314392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.908673873Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.940167103Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.940754338Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.940792237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.940806076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.940820306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} {"msg":"Crucible 31ce00dc-bdef-46ea-9570-2a78397198ca has session id: ab83f1f7-0a1d-4d9b-9248-82645ab04213","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.941348592Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Upstairs opts: Upstairs UUID: 31ce00dc-bdef-46ea-9570-2a78397198ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.941389791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Crucible stats registered with UUID: 31ce00dc-bdef-46ea-9570-2a78397198ca","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.94141305Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:50.94143291Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.941886828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.941921677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} {"msg":"31ce00dc-bdef-46ea-9570-2a78397198ca active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.941976746Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942002665Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"0","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942026754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"0","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942040794Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"1","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942055193Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"1","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942072023Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"2","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942086992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"2","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942103742Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942125202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942139901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"ds_connection connected from Ok(127.0.0.1:37988)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942227139Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"ds_connection connected from Ok(127.0.0.1:37191)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942261508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"ds_connection connected from Ok(127.0.0.1:62347)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.942452093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.943030968Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"1","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.943073017Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"2","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.943105866Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"0","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.94370133Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"1","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.943741349Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"0","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.943835606Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"2","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944340543Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.94443929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[0]R flush_numbers[0..12]: [310, 305, 325, 324, 313, 314, 301, 323, 293, 298, 324, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944509499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[0]R generation[0..12]: [169, 168, 173, 172, 170, 170, 165, 172, 159, 163, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944528858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944543248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[1]R flush_numbers[0..12]: [310, 305, 325, 324, 313, 314, 301, 323, 293, 298, 324, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944556718Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[1]R generation[0..12]: [169, 168, 173, 172, 170, 170, 165, 172, 159, 163, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944570037Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944583677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[2]R flush_numbers[0..12]: [310, 305, 325, 324, 313, 314, 301, 323, 293, 298, 324, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944597136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[2]R generation[0..12]: [169, 168, 173, 172, 170, 170, 165, 172, 159, 163, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944610486Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944624176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944636785Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944649475Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944661985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944677724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944697684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944709483Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"0","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944722303Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"1","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944734933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"client":"2","":"downstairs","session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944747813Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944759862Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"31ce00dc-bdef-46ea-9570-2a78397198ca is now active with session: ab83f1f7-0a1d-4d9b-9248-82645ab04213","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944772332Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944784021Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888,"session_id":"ab83f1f7-0a1d-4d9b-9248-82645ab04213"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944795891Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} {"msg":"Activated sub_volume 31ce00dc-bdef-46ea-9570-2a78397198ca","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:50.944806791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 16384 data: 13 15 16 15 02/30 Write block 338 len 4096 data: 18 03/30 Read block 555 len 32768 04/30 Write block 25 len 20480 data: 11 13 15 17 16 05/30 Flush 06/30 Write block 297 len 12288 data: 9 11 11 07/30 Read block 438 len 4096 08/30 Flush 09/30 Write block 233 len 12288 data: 7 7 12 10/30 Write block 179 len 40960 data: 16 12 10 11 11 13 10 8 11 10 11/30 Read block 427 len 12288 12/30 Flush 13/30 Write block 50 len 8192 data: 9 9 14/30 Read block 529 len 28672 15/30 Read block 81 len 16384 16/30 Flush 17/30 Write block 24 len 20480 data: 11 12 14 16 18 18/30 Read block 402 len 8192 19/30 Read block 533 len 36864 20/30 Read block 579 len 40960 21/30 Read block 121 len 20480 22/30 Write block 132 len 28672 data: 11 11 11 10 11 12 15 23/30 Read block 387 len 28672 24/30 Read block 259 len 24576 25/30 Flush 26/30 Write block 456 len 28672 data: 11 13 14 13 10 10 10 27/30 Read block 541 len 12288 28/30 Write block 154 len 4096 data: 9 29/30 Read block 43 len 12288 30/30 Write block 375 len 32768 data: 15 14 18 21 22 21 17 11 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:50.964670557Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1888} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 174 174 174 330 330 330 F F F 2 040-059 174 174 174 329 329 329 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 170 170 170 313 313 313 F F F 5 100-119 170 170 170 314 314 314 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 172 172 172 323 323 323 T T T 8 160-179 174 174 174 328 328 328 F F F 9 180-199 174 174 174 328 328 328 F F F 10 200-219 172 172 172 324 324 324 F F F 11 220-239 174 174 174 328 328 328 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 174 174 174 326 326 326 F F F 17 340-359 170 170 170 317 317 317 F F F 18 360-379 174 174 174 326 326 326 T T T 19 380-399 173 173 173 325 325 325 T T T 20 400-419 172 172 172 322 322 322 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 170 170 170 317 317 317 T T T 23 460-479 172 172 172 319 319 319 T T T 24 480-499 172 172 172 319 319 319 F F F 25 500-519 170 170 170 317 317 317 F F F 26 520-539 172 172 172 324 324 324 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 174, Max flush: 330 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 174 174 174 330 330 330 F F F 2 040-059 174 174 174 329 329 329 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 170 170 170 313 313 313 F F F 5 100-119 170 170 170 314 314 314 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 175 175 175 331 331 331 F F F 8 160-179 174 174 174 328 328 328 F F F 9 180-199 174 174 174 328 328 328 F F F 10 200-219 172 172 172 324 324 324 F F F 11 220-239 174 174 174 328 328 328 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 174 174 174 326 326 326 F F F 17 340-359 170 170 170 317 317 317 F F F 18 360-379 175 175 175 331 331 331 F F F 19 380-399 175 175 175 331 331 331 F F F 20 400-419 172 172 172 322 322 322 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 175 175 175 331 331 331 F F F 23 460-479 175 175 175 331 331 331 F F F 24 480-499 172 172 172 319 319 319 F F F 25 500-519 170 170 170 317 317 317 F F F 26 520-539 172 172 172 324 324 324 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 164 164 164 299 299 299 F F F Max gen: 175, Max flush: 331 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.226332255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.269786111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.270467463Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.270501612Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.270515051Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.270529201Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} {"msg":"Crucible 89abcb29-0534-447e-bc23-45b6e7a5e410 has session id: 9393d89a-7b73-4319-a635-9297a1ab5fa5","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271071627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Upstairs opts: Upstairs UUID: 89abcb29-0534-447e-bc23-45b6e7a5e410, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271103256Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Crucible stats registered with UUID: 89abcb29-0534-447e-bc23-45b6e7a5e410","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271117416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:55.271129905Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271638302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271666401Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} {"msg":"89abcb29-0534-447e-bc23-45b6e7a5e410 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.27172107Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271746809Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"0","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271771958Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"0","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271786038Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"1","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271800118Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"1","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271813217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271827567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271840506Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271853896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271867046Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"ds_connection connected from Ok(127.0.0.1:64939)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.271966353Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:55.272007962Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:55.272037221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42467)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.272065131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:11:55.27210011Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.272128949Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.272587597Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"1","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.27282438Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"0","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.273209541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"1","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.273323887Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"0","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.273717437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:11:55.273867093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.273611663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53638)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.273741609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.274839871Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.275445635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[0]R flush_numbers[0..12]: [310, 330, 329, 324, 313, 314, 330, 331, 328, 328, 324, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276059298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[0]R generation[0..12]: [169, 174, 174, 172, 170, 170, 174, 175, 174, 174, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276080858Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276100177Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[1]R flush_numbers[0..12]: [310, 330, 329, 324, 313, 314, 330, 331, 328, 328, 324, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276114207Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[1]R generation[0..12]: [169, 174, 174, 172, 170, 170, 174, 175, 174, 174, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276127566Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276141076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[2]R flush_numbers[0..12]: [310, 330, 329, 324, 313, 314, 330, 331, 328, 328, 324, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276154896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[2]R generation[0..12]: [169, 174, 174, 172, 170, 170, 174, 175, 174, 174, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276168405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276182365Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276195055Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276207844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276220424Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276244403Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"":"downstairs","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276257523Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276269263Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"0","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276282302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"1","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276297202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"client":"2","":"downstairs","session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276310722Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276322931Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"89abcb29-0534-447e-bc23-45b6e7a5e410 is now active with session: 9393d89a-7b73-4319-a635-9297a1ab5fa5","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276335441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.276347131Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896,"session_id":"9393d89a-7b73-4319-a635-9297a1ab5fa5"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.2763591Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} {"msg":"Activated sub_volume 89abcb29-0534-447e-bc23-45b6e7a5e410","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:05.2763779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 392 len 36864 data: 18 18 23 23 19 21 25 22 23 02/30 Read block 299 len 36864 03/30 Flush 04/30 Write block 114 len 24576 data: 17 17 16 17 15 14 05/30 Flush 06/30 Write block 33 len 20480 data: 16 14 16 15 13 07/30 Write block 442 len 8192 data: 8 7 08/30 Write block 44 len 36864 data: 15 15 12 13 10 9 10 10 10 09/30 Write block 351 len 24576 data: 13 12 9 12 11 11 10/30 Read block 343 len 24576 11/30 Write block 560 len 32768 data: 11 11 12 14 16 16 19 17 12/30 Read block 305 len 28672 13/30 Read block 285 len 16384 14/30 Write block 581 len 24576 data: 10 10 9 12 12 12 15/30 Write block 81 len 8192 data: 11 14 16/30 Write block 361 len 12288 data: 16 18 18 17/30 Read block 444 len 24576 18/30 Read block 322 len 4096 19/30 Flush 20/30 Flush 21/30 Write block 589 len 4096 data: 10 22/30 Write block 487 len 20480 data: 16 14 14 12 11 23/30 Write block 479 len 8192 data: 14 14 24/30 Write block 339 len 12288 data: 17 19 19 25/30 Write block 151 len 16384 data: 9 10 9 10 26/30 Write block 111 len 24576 data: 12 10 13 18 18 17 27/30 Read block 460 len 8192 28/30 Read block 597 len 8192 29/30 Read block 230 len 24576 30/30 Read block 478 len 16384 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:05.295926565Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1896} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 176 176 176 334 334 334 F F F 2 040-059 176 176 176 334 334 334 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 176 176 176 334 334 334 F F F 5 100-119 176 176 176 333 333 333 T T T 6 120-139 174 174 174 330 330 330 F F F 7 140-159 175 175 175 331 331 331 T T T 8 160-179 174 174 174 328 328 328 F F F 9 180-199 174 174 174 328 328 328 F F F 10 200-219 172 172 172 324 324 324 F F F 11 220-239 174 174 174 328 328 328 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 174 174 174 326 326 326 T T T 17 340-359 176 176 176 334 334 334 T T T 18 360-379 176 176 176 334 334 334 F F F 19 380-399 176 176 176 332 332 332 F F F 20 400-419 176 176 176 332 332 332 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 176 176 176 334 334 334 F F F 23 460-479 175 175 175 331 331 331 T T T 24 480-499 172 172 172 319 319 319 T T T 25 500-519 170 170 170 317 317 317 F F F 26 520-539 172 172 172 324 324 324 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 176 176 176 334 334 334 F F F 29 580-599 176 176 176 334 334 334 T T T Max gen: 176, Max flush: 334 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 F F F 1 020-039 176 176 176 334 334 334 F F F 2 040-059 176 176 176 334 334 334 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 176 176 176 334 334 334 F F F 5 100-119 177 177 177 335 335 335 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 177 177 177 335 335 335 F F F 8 160-179 174 174 174 328 328 328 F F F 9 180-199 174 174 174 328 328 328 F F F 10 200-219 172 172 172 324 324 324 F F F 11 220-239 174 174 174 328 328 328 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 177 177 177 335 335 335 F F F 17 340-359 177 177 177 335 335 335 F F F 18 360-379 176 176 176 334 334 334 F F F 19 380-399 176 176 176 332 332 332 F F F 20 400-419 176 176 176 332 332 332 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 176 176 176 334 334 334 F F F 23 460-479 177 177 177 335 335 335 F F F 24 480-499 177 177 177 335 335 335 F F F 25 500-519 170 170 170 317 317 317 F F F 26 520-539 172 172 172 324 324 324 F F F 27 540-559 170 170 170 317 317 317 F F F 28 560-579 176 176 176 334 334 334 F F F 29 580-599 177 177 177 335 335 335 F F F Max gen: 177, Max flush: 335 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.540460127Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.583819585Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.584510376Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.584545635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.584559395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.584573525Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} {"msg":"Crucible c206fdab-59e9-4fcb-864e-e017f51d1329 has session id: 6f01bc7a-b577-454e-b85e-8109960dbc9a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585155709Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Upstairs opts: Upstairs UUID: c206fdab-59e9-4fcb-864e-e017f51d1329, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585196728Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Crucible stats registered with UUID: c206fdab-59e9-4fcb-864e-e017f51d1329","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585213248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:19.585226527Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585587878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585611627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} {"msg":"c206fdab-59e9-4fcb-864e-e017f51d1329 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585655666Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585681045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"0","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585707515Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"0","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585722074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585736444Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585749304Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"2","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585771163Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"2","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585793282Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585817242Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585839511Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43893)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.58588206Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51246)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.585910409Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:19.585940798Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:19.586027646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:19.586111234Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.586142503Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.586667079Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"2","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.586694019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"0","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.587348081Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"2","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.587379371Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"0","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.587968025Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:19.588003874Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.587955429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59077)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.588018287Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.589077929Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.589681393Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[0]R flush_numbers[0..12]: [310, 334, 334, 324, 334, 335, 330, 335, 328, 328, 324, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590324816Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[0]R generation[0..12]: [169, 176, 176, 172, 176, 177, 174, 177, 174, 174, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590353535Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590370265Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[1]R flush_numbers[0..12]: [310, 334, 334, 324, 334, 335, 330, 335, 328, 328, 324, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590384214Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[1]R generation[0..12]: [169, 176, 176, 172, 176, 177, 174, 177, 174, 174, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590403034Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590417634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[2]R flush_numbers[0..12]: [310, 334, 334, 324, 334, 335, 330, 335, 328, 328, 324, 328]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590431093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[2]R generation[0..12]: [169, 176, 176, 172, 176, 177, 174, 177, 174, 174, 172, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590444323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590457752Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590470232Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590482902Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590495302Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590507941Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"":"downstairs","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590520781Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590532271Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"0","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.59054504Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"1","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590558Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"client":"2","":"downstairs","session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590570969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590582779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"c206fdab-59e9-4fcb-864e-e017f51d1329 is now active with session: 6f01bc7a-b577-454e-b85e-8109960dbc9a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590597079Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590608919Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904,"session_id":"6f01bc7a-b577-454e-b85e-8109960dbc9a"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:12:29.590620838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} {"msg":"Activated sub_volume c206fdab-59e9-4fcb-864e-e017f51d1329","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:29.590635448Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 578 len 16384 data: 14 12 10 11 03/30 Write block 100 len 16384 data: 21 22 20 20 04/30 Read block 50 len 40960 05/30 Read block 120 len 32768 06/30 Write block 500 len 16384 data: 8 8 7 11 07/30 Read block 503 len 12288 08/30 Read block 94 len 40960 09/30 Write block 174 len 8192 data: 12 15 10/30 Write block 157 len 24576 data: 14 14 14 10 9 11 11/30 Read block 446 len 12288 12/30 Write block 231 len 32768 data: 7 6 8 8 13 13 14 13 13/30 Read block 256 len 32768 14/30 Flush 15/30 Write block 554 len 4096 data: 14 16/30 Read block 92 len 20480 17/30 Write block 172 len 36864 data: 12 13 13 16 13 12 16 17 13 18/30 Write block 529 len 20480 data: 13 11 10 12 13 19/30 Write block 9 len 36864 data: 13 14 12 12 13 12 15 15 15 20/30 Read block 528 len 32768 21/30 Read block 393 len 4096 22/30 Write block 151 len 36864 data: 10 11 10 11 11 12 15 15 15 23/30 Read block 71 len 36864 24/30 Write block 448 len 24576 data: 8 7 12 12 15 14 25/30 Write block 493 len 12288 data: 14 17 17 26/30 Read block 70 len 20480 27/30 Write block 565 len 28672 data: 17 20 18 19 17 17 15 28/30 Read block 65 len 4096 29/30 Write block 195 len 32768 data: 9 10 7 7 8 8 11 12 30/30 Read block 225 len 8192 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 1 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:29.612078333Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1904} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 310 310 310 T T T 1 020-039 176 176 176 334 334 334 F F F 2 040-059 176 176 176 334 334 334 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 176 176 176 334 334 334 F F F 5 100-119 178 178 178 337 337 337 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 178 178 178 337 337 337 T T T 8 160-179 178 178 178 337 337 337 T T T 9 180-199 174 174 174 328 328 328 T T T 10 200-219 172 172 172 324 324 324 T T T 11 220-239 178 178 178 337 337 337 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 177 177 177 335 335 335 F F F 17 340-359 177 177 177 335 335 335 F F F 18 360-379 176 176 176 334 334 334 F F F 19 380-399 176 176 176 332 332 332 F F F 20 400-419 176 176 176 332 332 332 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 176 176 176 334 334 334 T T T 23 460-479 177 177 177 335 335 335 F F F 24 480-499 177 177 177 335 335 335 T T T 25 500-519 178 178 178 337 337 337 F F F 26 520-539 172 172 172 324 324 324 T T T 27 540-559 170 170 170 317 317 317 T T T 28 560-579 178 178 178 337 337 337 T T T 29 580-599 178 178 178 337 337 337 F F F Max gen: 178, Max flush: 337 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 338 338 338 F F F 1 020-039 176 176 176 334 334 334 F F F 2 040-059 176 176 176 334 334 334 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 176 176 176 334 334 334 F F F 5 100-119 178 178 178 337 337 337 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 179 179 179 338 338 338 F F F 8 160-179 179 179 179 338 338 338 F F F 9 180-199 179 179 179 338 338 338 F F F 10 200-219 179 179 179 338 338 338 F F F 11 220-239 178 178 178 337 337 337 F F F 12 240-259 172 172 172 324 324 324 F F F 13 260-279 168 168 168 308 308 308 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 170 170 170 317 317 317 F F F 16 320-339 177 177 177 335 335 335 F F F 17 340-359 177 177 177 335 335 335 F F F 18 360-379 176 176 176 334 334 334 F F F 19 380-399 176 176 176 332 332 332 F F F 20 400-419 176 176 176 332 332 332 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 179 179 179 338 338 338 F F F 23 460-479 177 177 177 335 335 335 F F F 24 480-499 179 179 179 338 338 338 F F F 25 500-519 178 178 178 337 337 337 F F F 26 520-539 179 179 179 338 338 338 F F F 27 540-559 179 179 179 338 338 338 F F F 28 560-579 179 179 179 338 338 338 F F F 29 580-599 178 178 178 337 337 337 F F F Max gen: 179, Max flush: 338 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.878649944Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.922181417Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.922914278Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.922962306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.922982916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.923005425Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} {"msg":"Crucible b3f0a6eb-d18a-430a-b76d-36985bd89228 has session id: 4b5d396c-7fdb-4cd5-bba5-bdcda580e034","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.92358313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Upstairs opts: Upstairs UUID: b3f0a6eb-d18a-430a-b76d-36985bd89228, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.923617529Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Crucible stats registered with UUID: b3f0a6eb-d18a-430a-b76d-36985bd89228","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.923633029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:43.923646048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924174504Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924203624Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} {"msg":"b3f0a6eb-d18a-430a-b76d-36985bd89228 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924253042Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924270202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"0","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924293531Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"0","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924311251Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.9243257Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.9243438Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"2","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924370389Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"2","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924398019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924426298Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924454177Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"ds_connection connected from Ok(127.0.0.1:51490)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924518385Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"ds_connection connected from Ok(127.0.0.1:58709)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924557834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:43.92472352Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:43.924750819Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:43.924767029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.924838447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.925188128Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"0","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.925220217Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"2","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.925986427Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"0","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.926118663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"2","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.926556172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:43.926648969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.926698419Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"ds_connection connected from Ok(127.0.0.1:49611)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.926893414Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.927703503Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.928373825Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[0]R flush_numbers[0..12]: [338, 334, 334, 324, 334, 337, 330, 338, 338, 338, 338, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929018708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[0]R generation[0..12]: [179, 176, 176, 172, 176, 178, 174, 179, 179, 179, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929047337Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929063887Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[1]R flush_numbers[0..12]: [338, 334, 334, 324, 334, 337, 330, 338, 338, 338, 338, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929078416Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[1]R generation[0..12]: [179, 176, 176, 172, 176, 178, 174, 179, 179, 179, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929092056Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929112155Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[2]R flush_numbers[0..12]: [338, 334, 334, 324, 334, 337, 330, 338, 338, 338, 338, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929129005Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[2]R generation[0..12]: [179, 176, 176, 172, 176, 178, 174, 179, 179, 179, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929152704Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929178984Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929195753Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929208843Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929224542Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929247372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929276701Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929290891Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"0","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.92930439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"1","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.92932477Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"client":"2","":"downstairs","session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929349579Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929369948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"b3f0a6eb-d18a-430a-b76d-36985bd89228 is now active with session: 4b5d396c-7fdb-4cd5-bba5-bdcda580e034","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929383248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929406447Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912,"session_id":"4b5d396c-7fdb-4cd5-bba5-bdcda580e034"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929430187Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} {"msg":"Activated sub_volume b3f0a6eb-d18a-430a-b76d-36985bd89228","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:53.929447997Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 397 len 36864 data: 22 26 23 24 20 18 17 15 16 02/30 Write block 247 len 20480 data: 16 17 18 18 18 03/30 Read block 376 len 12288 04/30 Read block 208 len 28672 05/30 Write block 202 len 12288 data: 13 10 8 06/30 Read block 251 len 12288 07/30 Read block 551 len 12288 08/30 Flush 09/30 Write block 242 len 24576 data: 10 12 9 12 13 17 10/30 Read block 564 len 4096 11/30 Read block 530 len 16384 12/30 Write block 167 len 40960 data: 11 13 13 15 16 13 14 14 17 14 13/30 Read block 396 len 32768 14/30 Flush 15/30 Write block 272 len 12288 data: 14 15 17 16/30 Write block 302 len 4096 data: 12 17/30 Write block 359 len 36864 data: 16 16 17 19 19 15 14 16 17 18/30 Read block 11 len 32768 19/30 Write block 260 len 28672 data: 13 18 17 15 13 12 15 20/30 Write block 49 len 36864 data: 10 11 11 11 11 13 14 12 15 21/30 Read block 264 len 8192 22/30 Flush 23/30 Write block 148 len 12288 data: 10 7 7 24/30 Read block 45 len 12288 25/30 Read block 294 len 24576 26/30 Write block 1 len 36864 data: 3 5 5 9 10 13 11 9 14 27/30 Write block 86 len 24576 data: 15 17 17 18 17 16 28/30 Read block 182 len 40960 29/30 Read block 359 len 24576 30/30 Write block 477 len 40960 data: 12 12 15 15 14 14 13 13 13 17 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:53.95057437Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1912} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 338 338 338 T T T 1 020-039 176 176 176 334 334 334 F F F 2 040-059 180 180 180 341 341 341 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 176 176 176 334 334 334 T T T 5 100-119 178 178 178 337 337 337 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 179 179 179 338 338 338 T T T 8 160-179 180 180 180 340 340 340 F F F 9 180-199 179 179 179 338 338 338 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 178 178 178 337 337 337 F F F 12 240-259 180 180 180 340 340 340 F F F 13 260-279 180 180 180 341 341 341 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 180 180 180 341 341 341 F F F 16 320-339 177 177 177 335 335 335 F F F 17 340-359 180 180 180 341 341 341 F F F 18 360-379 180 180 180 341 341 341 F F F 19 380-399 180 180 180 339 339 339 F F F 20 400-419 180 180 180 339 339 339 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 179 179 179 338 338 338 F F F 23 460-479 177 177 177 335 335 335 T T T 24 480-499 179 179 179 338 338 338 T T T 25 500-519 178 178 178 337 337 337 F F F 26 520-539 179 179 179 338 338 338 F F F 27 540-559 179 179 179 338 338 338 F F F 28 560-579 179 179 179 338 338 338 F F F 29 580-599 178 178 178 337 337 337 F F F Max gen: 180, Max flush: 341 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 342 342 342 F F F 1 020-039 176 176 176 334 334 334 F F F 2 040-059 180 180 180 341 341 341 F F F 3 060-079 172 172 172 324 324 324 F F F 4 080-099 181 181 181 342 342 342 F F F 5 100-119 178 178 178 337 337 337 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 180 180 180 340 340 340 F F F 9 180-199 179 179 179 338 338 338 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 178 178 178 337 337 337 F F F 12 240-259 180 180 180 340 340 340 F F F 13 260-279 180 180 180 341 341 341 F F F 14 280-299 174 174 174 327 327 327 F F F 15 300-319 180 180 180 341 341 341 F F F 16 320-339 177 177 177 335 335 335 F F F 17 340-359 180 180 180 341 341 341 F F F 18 360-379 180 180 180 341 341 341 F F F 19 380-399 180 180 180 339 339 339 F F F 20 400-419 180 180 180 339 339 339 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 179 179 179 338 338 338 F F F 23 460-479 181 181 181 342 342 342 F F F 24 480-499 181 181 181 342 342 342 F F F 25 500-519 178 178 178 337 337 337 F F F 26 520-539 179 179 179 338 338 338 F F F 27 540-559 179 179 179 338 338 338 F F F 28 560-579 179 179 179 338 338 338 F F F 29 580-599 178 178 178 337 337 337 F F F Max gen: 181, Max flush: 342 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.186463807Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.229677248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.23034948Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.23038314Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.230401169Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.230416019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} {"msg":"Crucible b18d14fa-cec6-4bd7-9a5f-3fb3af3dcd6b has session id: 5f8df646-3fe5-4c6c-bbcd-11f653cf8927","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.230997483Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Upstairs opts: Upstairs UUID: b18d14fa-cec6-4bd7-9a5f-3fb3af3dcd6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231041872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Crucible stats registered with UUID: b18d14fa-cec6-4bd7-9a5f-3fb3af3dcd6b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231065192Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:58.231085931Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231577938Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231602498Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} {"msg":"b18d14fa-cec6-4bd7-9a5f-3fb3af3dcd6b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231668076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231684705Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"0","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231706285Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"0","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231719864Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231733984Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231747184Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"2","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231769743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"2","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231792353Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231815902Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.231838191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"ds_connection connected from Ok(127.0.0.1:56600)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.23186458Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"ds_connection connected from Ok(127.0.0.1:45306)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.23189489Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:58.231924009Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:58.231953278Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:12:58.232086495Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.23226027Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.232699818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"0","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.232727928Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"2","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.233322922Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"0","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.233459349Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"2","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.233962805Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:12:58.234117221Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.2330969Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"ds_connection connected from Ok(127.0.0.1:40461)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.233197737Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.23425117Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.234882833Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[0]R flush_numbers[0..12]: [342, 334, 341, 324, 342, 337, 330, 342, 340, 338, 339, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235530466Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[0]R generation[0..12]: [181, 176, 180, 172, 181, 178, 174, 181, 180, 179, 180, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235555775Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235571605Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[1]R flush_numbers[0..12]: [342, 334, 341, 324, 342, 337, 330, 342, 340, 338, 339, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235586125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[1]R generation[0..12]: [181, 176, 180, 172, 181, 178, 174, 181, 180, 179, 180, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235600374Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235614344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[2]R flush_numbers[0..12]: [342, 334, 341, 324, 342, 337, 330, 342, 340, 338, 339, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235627924Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[2]R generation[0..12]: [181, 176, 180, 172, 181, 178, 174, 181, 180, 179, 180, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235641543Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235659903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235673002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235686052Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Next flush: 343","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235698822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235711501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"":"downstairs","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235724341Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.23573609Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"0","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.23574933Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"1","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.23576219Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"client":"2","":"downstairs","session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.23577534Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235787379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"b18d14fa-cec6-4bd7-9a5f-3fb3af3dcd6b is now active with session: 5f8df646-3fe5-4c6c-bbcd-11f653cf8927","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235799979Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235811859Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920,"session_id":"5f8df646-3fe5-4c6c-bbcd-11f653cf8927"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235824048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} {"msg":"Activated sub_volume b18d14fa-cec6-4bd7-9a5f-3fb3af3dcd6b","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:08.235835148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 447 len 24576 data: 10 9 8 13 13 16 02/30 Read block 223 len 32768 03/30 Write block 288 len 8192 data: 4 7 04/30 Write block 29 len 16384 data: 17 15 16 17 05/30 Write block 56 len 28672 data: 13 16 12 14 15 10 15 06/30 Read block 427 len 24576 07/30 Write block 242 len 32768 data: 11 13 10 13 14 18 18 19 08/30 Read block 545 len 36864 09/30 Write block 402 len 12288 data: 19 18 16 10/30 Read block 169 len 8192 11/30 Read block 267 len 16384 12/30 Write block 472 len 36864 data: 9 10 10 15 12 13 13 16 16 13/30 Write block 565 len 32768 data: 18 21 19 20 18 18 16 15 14/30 Read block 198 len 16384 15/30 Write block 542 len 40960 data: 16 18 17 16 14 20 20 18 18 18 16/30 Read block 483 len 36864 17/30 Write block 287 len 28672 data: 6 5 8 9 8 7 10 18/30 Read block 343 len 20480 19/30 Read block 274 len 12288 20/30 Write block 532 len 36864 data: 13 14 13 16 18 17 17 18 14 21/30 Write block 329 len 36864 data: 12 12 14 15 16 12 13 14 15 22/30 Read block 166 len 36864 23/30 Write block 463 len 32768 data: 11 10 9 10 9 7 9 11 24/30 Write block 588 len 40960 data: 11 11 12 10 10 9 9 8 10 8 25/30 Write block 90 len 4096 data: 18 26/30 Read block 361 len 20480 27/30 Write block 73 len 32768 data: 13 18 19 19 18 19 14 11 28/30 Read block 83 len 24576 29/30 Read block 324 len 20480 30/30 Write block 398 len 16384 data: 27 24 25 21 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:08.257674963Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1920} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 342 342 342 F F F 1 020-039 176 176 176 334 334 334 T T T 2 040-059 180 180 180 341 341 341 T T T 3 060-079 172 172 172 324 324 324 T T T 4 080-099 181 181 181 342 342 342 T T T 5 100-119 178 178 178 337 337 337 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 180 180 180 340 340 340 F F F 9 180-199 179 179 179 338 338 338 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 178 178 178 337 337 337 F F F 12 240-259 180 180 180 340 340 340 T T T 13 260-279 180 180 180 341 341 341 F F F 14 280-299 174 174 174 327 327 327 T T T 15 300-319 180 180 180 341 341 341 F F F 16 320-339 177 177 177 335 335 335 T T T 17 340-359 180 180 180 341 341 341 F F F 18 360-379 180 180 180 341 341 341 F F F 19 380-399 180 180 180 339 339 339 T T T 20 400-419 180 180 180 339 339 339 T T T 21 420-439 170 170 170 311 311 311 F F F 22 440-459 179 179 179 338 338 338 T T T 23 460-479 181 181 181 342 342 342 T T T 24 480-499 181 181 181 342 342 342 T T T 25 500-519 178 178 178 337 337 337 F F F 26 520-539 179 179 179 338 338 338 T T T 27 540-559 179 179 179 338 338 338 T T T 28 560-579 179 179 179 338 338 338 T T T 29 580-599 178 178 178 337 337 337 T T T Max gen: 181, Max flush: 342 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 342 342 342 F F F 1 020-039 182 182 182 343 343 343 F F F 2 040-059 182 182 182 343 343 343 F F F 3 060-079 182 182 182 343 343 343 F F F 4 080-099 182 182 182 343 343 343 F F F 5 100-119 178 178 178 337 337 337 F F F 6 120-139 174 174 174 330 330 330 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 180 180 180 340 340 340 F F F 9 180-199 179 179 179 338 338 338 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 178 178 178 337 337 337 F F F 12 240-259 182 182 182 343 343 343 F F F 13 260-279 180 180 180 341 341 341 F F F 14 280-299 182 182 182 343 343 343 F F F 15 300-319 180 180 180 341 341 341 F F F 16 320-339 182 182 182 343 343 343 F F F 17 340-359 180 180 180 341 341 341 F F F 18 360-379 180 180 180 341 341 341 F F F 19 380-399 182 182 182 343 343 343 F F F 20 400-419 182 182 182 343 343 343 F F F 21 420-439 170 170 170 311 311 311 F F F 22 440-459 182 182 182 343 343 343 F F F 23 460-479 182 182 182 343 343 343 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 178 178 178 337 337 337 F F F 26 520-539 182 182 182 343 343 343 F F F 27 540-559 182 182 182 343 343 343 F F F 28 560-579 182 182 182 343 343 343 F F F 29 580-599 182 182 182 343 343 343 F F F Max gen: 182, Max flush: 343 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.557450007Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.601080708Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.601737581Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.60177339Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.601787769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.601802039Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} {"msg":"Crucible 8ef81492-aca7-41ec-8323-72b3dc675fd9 has session id: 38653f23-3737-496e-835c-231b2de2aad1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.602473421Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Upstairs opts: Upstairs UUID: 8ef81492-aca7-41ec-8323-72b3dc675fd9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.60251985Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Crucible stats registered with UUID: 8ef81492-aca7-41ec-8323-72b3dc675fd9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.602543799Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:22.602564989Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.602947959Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.602973978Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} {"msg":"8ef81492-aca7-41ec-8323-72b3dc675fd9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603033866Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603051116Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"0","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603076565Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"0","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603091065Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"1","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603108844Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"1","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603122474Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603136244Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603149074Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603165473Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603187802Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:22.60326372Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:22.603299459Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43720)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603326479Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:22.603461445Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603490714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59506)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603525764Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.603911423Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"0","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.604082589Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"1","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.604521667Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"0","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.604664863Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"1","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.605096722Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:22.605240648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.603716492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59717)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.60378077Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.604846672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.605527814Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[0]R flush_numbers[0..12]: [342, 343, 343, 343, 343, 337, 330, 342, 340, 338, 339, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606117078Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[0]R generation[0..12]: [181, 182, 182, 182, 182, 178, 174, 181, 180, 179, 180, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606146137Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606163327Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[1]R flush_numbers[0..12]: [342, 343, 343, 343, 343, 337, 330, 342, 340, 338, 339, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606177687Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[1]R generation[0..12]: [181, 182, 182, 182, 182, 178, 174, 181, 180, 179, 180, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606191126Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606205136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[2]R flush_numbers[0..12]: [342, 343, 343, 343, 343, 337, 330, 342, 340, 338, 339, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606223246Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[2]R generation[0..12]: [181, 182, 182, 182, 182, 178, 174, 181, 180, 179, 180, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606236995Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606250675Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Max found gen is 183","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606263284Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Generation requested: 184 >= found:183","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606276164Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606288784Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606301403Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"":"downstairs","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606314443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606326243Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"0","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606339172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"1","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606351852Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"client":"2","":"downstairs","session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606364502Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606376411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"8ef81492-aca7-41ec-8323-72b3dc675fd9 is now active with session: 38653f23-3737-496e-835c-231b2de2aad1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606388761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.606400441Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928,"session_id":"Wait for a query_work_queue command to finish before sending IO 38653f23-3737-496e-835c-231b2de2aad1"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.60642072Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} {"msg":"Activated sub_volume 8ef81492-aca7-41ec-8323-72b3dc675fd9","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:32.6064318Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 20480 data: 22 23 21 21 19 02/30 Flush 03/30 Write block 93 len 40960 data: 16 14 14 16 16 17 18 23 24 22 04/30 Read block 485 len 36864 05/30 Write block 532 len 40960 data: 14 15 14 17 19 18 18 19 15 14 06/30 Read block 0 len 20480 07/30 Flush 08/30 Write block 220 len 8192 data: 6 10 09/30 Flush 10/30 Write block 511 len 28672 data: 17 15 15 17 13 12 12 11/30 Write block 555 len 28672 data: 16 12 11 10 9 12 12 12/30 Write block 29 len 8192 data: 18 16 13/30 Write block 129 len 12288 data: 13 9 9 14/30 Write block 425 len 40960 data: 15 12 9 10 10 11 14 10 11 11 15/30 Write block 33 len 28672 data: 17 15 17 16 14 10 9 16/30 Read block 237 len 28672 17/30 Flush 18/30 Write block 580 len 16384 data: 11 12 11 10 19/30 Write block 458 len 32768 data: 15 14 11 11 11 12 11 10 20/30 Read block 429 len 8192 21/30 Write block 528 len 40960 data: 14 14 12 11 15 16 15 18 20 19 22/30 Read block 489 len 28672 23/30 Write block 421 len 28672 data: 14 14 17 16 16 13 10 24/30 Read block 86 len 36864 25/30 Write block 335 len 36864 data: 14 15 16 19 18 20 20 19 18 26/30 Read block 363 len 24576 27/30 Write block 253 len 24576 data: 16 12 11 11 11 9 28/30 Read block 67 len 20480 29/30 Read block 534 len 28672 30/30 Read block 349 len 32768 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:32.627854276Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 342 342 342 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 182 182 182 343 343 343 F F F 3 060-079 182 182 182 343 343 343 F F F 4 080-099 184 184 184 345 345 345 F F F 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 180 180 180 340 340 340 F F F 9 180-199 179 179 179 338 338 338 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 182 182 182 343 343 343 T T T 13 260-279 180 180 180 341 341 341 F F F 14 280-299 182 182 182 343 343 343 F F F 15 300-319 180 180 180 341 341 341 F F F 16 320-339 182 182 182 343 343 343 T T T 17 340-359 180 180 180 341 341 341 T T T 18 360-379 180 180 180 341 341 341 F F F 19 380-399 182 182 182 343 343 343 F F F 20 400-419 182 182 182 343 343 343 F F F 21 420-439 184 184 184 347 347 347 T T T 22 440-459 182 182 182 343 343 343 T T T 23 460-479 182 182 182 343 343 343 T T T 24 480-499 182 182 182 343 343 343 F F F 25 500-519 184 184 184 347 347 347 F F F 26 520-539 184 184 184 345 345 345 T T T 27 540-559 184 184 184 347 347 347 F F F 28 560-579 184 184 184 347 347 347 F F F 29 580-599 182 182 182 343 343 343 T T T Max gen: 184, Max flush: 347 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 342 342 342 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 182 182 182 343 343 343 F F F 3 060-079 182 182 182 343 343 343 F F F 4 080-099 184 184 184 345 345 345 F F F 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 180 180 180 340 340 340 F F F 9 180-199 179 179 179 338 338 338 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 180 180 180 341 341 341 F F F 14 280-299 182 182 182 343 343 343 F F F 15 300-319 180 180 180 341 341 341 F F F 16 320-339 185 185 185 348 348 348 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 180 180 180 341 341 341 F F F 19 380-399 182 182 182 343 343 343 F F F 20 400-419 182 182 182 343 343 343 F F F 21 420-439 185 185 185 348 348 348 F F F 22 440-459 185 185 185 348 348 348 F F F 23 460-479 185 185 185 348 348 348 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 184 184 184 347 347 347 F F F 26 520-539 185 185 185 348 348 348 F F F 27 540-559 184 184 184 347 347 347 F F F 28 560-579 184 184 184 347 347 347 F F F 29 580-599 185 185 185 348 348 348 F F F Max gen: 185, Max flush: 348 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.883105464Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:46.924990951Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.92537676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.956444972Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.957073405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.957102745Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.957115834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.957129744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} {"msg":"Crucible bb8f983e-3143-423d-88fa-5d313bab2d3a has session id: bfc0d565-0530-48c9-a518-6970c3dcf417","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.957746258Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Upstairs opts: Upstairs UUID: bb8f983e-3143-423d-88fa-5d313bab2d3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.957781237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Crucible stats registered with UUID: bb8f983e-3143-423d-88fa-5d313bab2d3a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.957797577Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:46.957818866Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958249194Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958274344Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} {"msg":"bb8f983e-3143-423d-88fa-5d313bab2d3a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958336822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958353932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"0","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958377061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"0","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958390651Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"1","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.95840793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"1","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.95842746Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"2","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958445139Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"2","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958464539Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958478999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958498368Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"ds_connection connected from Ok(127.0.0.1:45871)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958583686Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"ds_connection connected from Ok(127.0.0.1:36575)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958824749Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"ds_connection connected from Ok(127.0.0.1:37064)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.958866148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.959408164Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"2","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.959437633Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"1","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.95955992Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"0","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960039027Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"2","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960184903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"1","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960225322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"0","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960658341Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960768818Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[0]R flush_numbers[0..12]: [342, 347, 343, 343, 345, 345, 347, 342, 340, 338, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960884635Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[0]R generation[0..12]: [181, 184, 182, 182, 184, 184, 184, 181, 180, 179, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960905255Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960919634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[1]R flush_numbers[0..12]: [342, 347, 343, 343, 345, 345, 347, 342, 340, 338, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960933434Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[1]R generation[0..12]: [181, 184, 182, 182, 184, 184, 184, 181, 180, 179, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960946883Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960960523Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[2]R flush_numbers[0..12]: [342, 347, 343, 343, 345, 345, 347, 342, 340, 338, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960973773Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[2]R generation[0..12]: [181, 184, 182, 182, 184, 184, 184, 181, 180, 179, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.960987002Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961000402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961013112Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961025791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961047861Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.96106108Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"":"downstairs","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.96107392Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.96108547Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"0","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961098379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"1","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961111149Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"client":"2","":"downstairs","session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961123939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961135918Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"bb8f983e-3143-423d-88fa-5d313bab2d3a is now active with session: bfc0d565-0530-48c9-a518-6970c3dcf417","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961148408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961160068Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936,"session_id":"bfc0d565-0530-48c9-a518-6970c3dcf417"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-19T17:13:46.961171828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} {"msg":"Activated sub_volume bb8f983e-3143-423d-88fa-5d313bab2d3a","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:46.961185277Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 543 len 20480 data: 19 18 17 15 21 03/30 Read block 455 len 16384 04/30 Write block 76 len 36864 data: 20 19 20 15 12 12 15 17 16 05/30 Write block 587 len 12288 data: 14 12 12 06/30 Read block 293 len 24576 07/30 Read block 447 len 20480 08/30 Read block 558 len 36864 09/30 Write block 1 len 8192 data: 4 6 10/30 Write block 435 len 32768 data: 10 12 11 11 10 9 7 9 11/30 Write block 456 len 28672 data: 12 14 16 15 12 12 12 12/30 Read block 383 len 20480 13/30 Read block 203 len 20480 14/30 Read block 433 len 24576 15/30 Read block 40 len 12288 16/30 Write block 283 len 12288 data: 10 9 8 17/30 Read block 316 len 20480 18/30 Write block 276 len 12288 data: 17 16 15 19/30 Read block 26 len 16384 20/30 Write block 56 len 28672 data: 14 17 13 15 16 11 16 21/30 Write block 45 len 8192 data: 16 13 22/30 Write block 534 len 4096 data: 16 23/30 Flush 24/30 Write block 538 len 12288 data: 19 20 16 25/30 Flush 26/30 Write block 67 len 4096 data: 15 27/30 Read block 531 len 12288 28/30 Write block 283 len 20480 data: 11 10 9 7 7 29/30 Write block 177 len 24576 data: 13 17 18 14 11 12 30/30 Read block 412 len 36864 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:46.980479939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1936} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 350 350 350 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 186 186 186 350 350 350 F F F 3 060-079 186 186 186 350 350 350 T T T 4 080-099 186 186 186 350 350 350 F F F 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 180 180 180 340 340 340 T T T 9 180-199 179 179 179 338 338 338 T T T 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 186 186 186 350 350 350 F F F 14 280-299 186 186 186 350 350 350 T T T 15 300-319 180 180 180 341 341 341 F F F 16 320-339 185 185 185 348 348 348 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 180 180 180 341 341 341 F F F 19 380-399 182 182 182 343 343 343 F F F 20 400-419 182 182 182 343 343 343 F F F 21 420-439 186 186 186 350 350 350 F F F 22 440-459 186 186 186 350 350 350 F F F 23 460-479 186 186 186 350 350 350 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 184 184 184 347 347 347 F F F 26 520-539 186 186 186 351 351 351 F F F 27 540-559 186 186 186 351 351 351 F F F 28 560-579 184 184 184 347 347 347 F F F 29 580-599 186 186 186 350 350 350 F F F Max gen: 186, Max flush: 351 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 350 350 350 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 186 186 186 350 350 350 F F F 3 060-079 187 187 187 352 352 352 F F F 4 080-099 186 186 186 350 350 350 F F F 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 187 187 187 352 352 352 F F F 9 180-199 187 187 187 352 352 352 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 186 186 186 350 350 350 F F F 14 280-299 187 187 187 352 352 352 F F F 15 300-319 180 180 180 341 341 341 F F F 16 320-339 185 185 185 348 348 348 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 180 180 180 341 341 341 F F F 19 380-399 182 182 182 343 343 343 F F F 20 400-419 182 182 182 343 343 343 F F F 21 420-439 186 186 186 350 350 350 F F F 22 440-459 186 186 186 350 350 350 F F F 23 460-479 186 186 186 350 350 350 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 184 184 184 347 347 347 F F F 26 520-539 186 186 186 351 351 351 F F F 27 540-559 186 186 186 351 351 351 F F F 28 560-579 184 184 184 347 347 347 F F F 29 580-599 186 186 186 350 350 350 F F F Max gen: 187, Max flush: 352 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.244699409Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.288504835Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.289168828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.289202777Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.289216487Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.289230466Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} {"msg":"Crucible 43d9addb-555e-45d6-8e43-84fa59e74252 has session id: c6b1dfbb-7cd3-41bd-9196-78bd154556e7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.289916158Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Upstairs opts: Upstairs UUID: 43d9addb-555e-45d6-8e43-84fa59e74252, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.289955867Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Crucible stats registered with UUID: 43d9addb-555e-45d6-8e43-84fa59e74252","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.289972087Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:51.289984456Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290441544Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290476323Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} {"msg":"43d9addb-555e-45d6-8e43-84fa59e74252 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290532652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290559641Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"0","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.2905845Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"0","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.29060093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.29061627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290638919Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"2","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290664838Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"2","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290687148Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290710037Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290733576Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:42802)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290759916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56444)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.290786425Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:51.290813545Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:51.290848253Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:13:51.29096652Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.2909906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.291527896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"0","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.291556645Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"2","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.292097541Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"2","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.292212197Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"0","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.292758023Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:13:51.292792082Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.292596411Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34103)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.292661029Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.293705061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.294341414Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[0]R flush_numbers[0..12]: [350, 347, 350, 352, 350, 345, 347, 342, 352, 352, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.294987237Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[0]R generation[0..12]: [186, 184, 186, 187, 186, 184, 184, 181, 187, 187, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295020916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295037506Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[1]R flush_numbers[0..12]: [350, 347, 350, 352, 350, 345, 347, 342, 352, 352, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295052156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[1]R generation[0..12]: [186, 184, 186, 187, 186, 184, 184, 181, 187, 187, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295066165Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295080105Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[2]R flush_numbers[0..12]: [350, 347, 350, 352, 350, 345, 347, 342, 352, 352, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295093715Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[2]R generation[0..12]: [186, 184, 186, 187, 186, 184, 184, 181, 187, 187, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295107334Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295121304Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295134094Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295147293Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Next flush: 353","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295160113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295172783Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"":"downstairs","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295185812Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295197582Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"0","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295212811Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"1","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295226061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"client":"2","":"downstairs","session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295239071Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295251001Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"43d9addb-555e-45d6-8e43-84fa59e74252 is now active with session: c6b1dfbb-7cd3-41bd-9196-78bd154556e7","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.29526359Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.29527538Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944,"session_id":"c6b1dfbb-7cd3-41bd-9196-78bd154556e7"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:14:01.295287429Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} {"msg":"Activated sub_volume 43d9addb-555e-45d6-8e43-84fa59e74252","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:01.295303439Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 550 len 40960 data: 19 19 14 17 15 17 13 12 11 10 02/30 Write block 471 len 20480 data: 10 10 11 11 16 03/30 Write block 59 len 28672 data: 16 17 12 17 16 15 15 04/30 Read block 391 len 36864 05/30 Read block 588 len 24576 06/30 Read block 501 len 32768 07/30 Write block 367 len 40960 data: 18 16 12 15 15 15 15 18 16 15 08/30 Write block 270 len 32768 data: 18 12 15 16 18 19 18 17 09/30 Write block 171 len 20480 data: 17 14 15 15 18 10/30 Read block 182 len 36864 11/30 Write block 465 len 4096 data: 11 12/30 Write block 386 len 40960 data: 9 9 6 8 10 15 19 19 24 24 13/30 Write block 174 len 32768 data: 16 19 15 14 18 19 15 12 14/30 Write block 553 len 32768 data: 18 16 18 14 13 12 11 13 15/30 Read block 401 len 20480 16/30 Write block 68 len 28672 data: 16 15 11 12 10 14 19 17/30 Write block 75 len 28672 data: 20 21 20 21 16 13 13 18/30 Write block 568 len 32768 data: 21 19 19 17 16 15 14 14 19/30 Write block 454 len 32768 data: 15 14 13 15 17 16 13 13 20/30 Read block 435 len 12288 21/30 Read block 582 len 20480 22/30 Write block 505 len 24576 data: 10 8 9 14 13 14 23/30 Read block 374 len 4096 24/30 Write block 419 len 28672 data: 16 17 15 15 18 17 17 25/30 Write block 45 len 4096 data: 17 26/30 Write block 414 len 24576 data: 11 12 14 14 16 17 27/30 Write block 95 len 16384 data: 15 17 17 18 28/30 Write block 318 len 32768 data: 14 16 15 18 13 14 14 15 29/30 Read block 414 len 12288 30/30 Write block 462 len 40960 data: 13 13 12 12 11 10 8 10 12 11 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:01.315737501Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1944} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 350 350 350 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 186 186 186 350 350 350 T T T 3 060-079 187 187 187 352 352 352 T T T 4 080-099 186 186 186 350 350 350 T T T 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 187 187 187 352 352 352 T T T 9 180-199 187 187 187 352 352 352 T T T 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 186 186 186 350 350 350 T T T 14 280-299 187 187 187 352 352 352 F F F 15 300-319 180 180 180 341 341 341 T T T 16 320-339 185 185 185 348 348 348 T T T 17 340-359 185 185 185 348 348 348 F F F 18 360-379 180 180 180 341 341 341 T T T 19 380-399 182 182 182 343 343 343 T T T 20 400-419 182 182 182 343 343 343 T T T 21 420-439 186 186 186 350 350 350 T T T 22 440-459 186 186 186 350 350 350 T T T 23 460-479 186 186 186 350 350 350 T T T 24 480-499 182 182 182 343 343 343 F F F 25 500-519 184 184 184 347 347 347 T T T 26 520-539 186 186 186 351 351 351 F F F 27 540-559 186 186 186 351 351 351 T T T 28 560-579 184 184 184 347 347 347 T T T 29 580-599 186 186 186 350 350 350 F F F Max gen: 187, Max flush: 352 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 350 350 350 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 188 188 188 353 353 353 F F F 3 060-079 188 188 188 353 353 353 F F F 4 080-099 188 188 188 353 353 353 F F F 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 188 188 188 353 353 353 F F F 9 180-199 188 188 188 353 353 353 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 188 188 188 353 353 353 F F F 14 280-299 187 187 187 352 352 352 F F F 15 300-319 188 188 188 353 353 353 F F F 16 320-339 188 188 188 353 353 353 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 188 188 188 353 353 353 F F F 19 380-399 188 188 188 353 353 353 F F F 20 400-419 188 188 188 353 353 353 F F F 21 420-439 188 188 188 353 353 353 F F F 22 440-459 188 188 188 353 353 353 F F F 23 460-479 188 188 188 353 353 353 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 188 188 188 353 353 353 F F F 26 520-539 186 186 186 351 351 351 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 188 188 188 353 353 353 F F F 29 580-599 186 186 186 350 350 350 F F F Max gen: 188, Max flush: 353 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.613412903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:15.655735878Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.656183806Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.686981315Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.687602759Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.687631688Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.687644358Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.687658417Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} {"msg":"Crucible 0059b322-59a6-4a25-b15c-49981141c048 has session id: 6fedf6d0-329c-4f7b-bb39-9d731015bad4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688206503Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Upstairs opts: Upstairs UUID: 0059b322-59a6-4a25-b15c-49981141c048, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688248422Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Crucible stats registered with UUID: 0059b322-59a6-4a25-b15c-49981141c048","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688265841Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:15.688278931Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.68869378Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688727999Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} {"msg":"0059b322-59a6-4a25-b15c-49981141c048 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688781857Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688813267Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"0","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688856575Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"0","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688890395Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"1","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688927104Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"1","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688955113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"2","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.688986872Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"2","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.689019461Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.68904843Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.68908036Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56054)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.689110699Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41813)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.689138798Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62596)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.689313634Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.689772202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"2","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.689951957Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"1","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.690076593Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"0","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.690510292Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"2","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.690550101Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"1","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.690679118Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"0","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691097627Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691162995Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[0]R flush_numbers[0..12]: [350, 347, 353, 353, 353, 345, 347, 342, 353, 353, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691315271Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[0]R generation[0..12]: [186, 184, 188, 188, 188, 184, 184, 181, 188, 188, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.69133567Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.69135373Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[1]R flush_numbers[0..12]: [350, 347, 353, 353, 353, 345, 347, 342, 353, 353, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691367769Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[1]R generation[0..12]: [186, 184, 188, 188, 188, 184, 184, 181, 188, 188, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691381059Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691394499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[2]R flush_numbers[0..12]: [350, 347, 353, 353, 353, 345, 347, 342, 353, 353, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691407668Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[2]R generation[0..12]: [186, 184, 188, 188, 188, 184, 184, 181, 188, 188, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691420868Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691434518Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691447147Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691459917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Next flush: 354","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691472537Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691491906Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"":"downstairs","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691504986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691517096Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"0","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691530295Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"1","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691545755Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"client":"2","":"downstairs","session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691559084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691571084Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"0059b322-59a6-4a25-b15c-49981141c048 is now active with session: 6fedf6d0-329c-4f7b-bb39-9d731015bad4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691583524Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691595113Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952,"session_id":"6fedf6d0-329c-4f7b-bb39-9d731015bad4"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691606793Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} {"msg":"Activated sub_volume 0059b322-59a6-4a25-b15c-49981141c048","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:15.691617673Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 432 len 8192 data: 11 12 02/30 Read block 25 len 16384 03/30 Write block 507 len 20480 data: 10 15 14 15 18 04/30 Read block 49 len 4096 05/30 Read block 247 len 32768 06/30 Write block 67 len 24576 data: 16 17 16 12 13 11 07/30 Read block 565 len 12288 08/30 Write block 529 len 12288 data: 15 13 12 09/30 Read block 385 len 12288 10/30 Write block 528 len 8192 data: 15 16 11/30 Read block 259 len 8192 12/30 Write block 593 len 24576 data: 10 10 9 11 9 6 13/30 Read block 418 len 4096 14/30 Write block 294 len 8192 data: 12 11 15/30 Read block 89 len 8192 16/30 Write block 265 len 4096 data: 13 17/30 Read block 98 len 8192 18/30 Read block 485 len 16384 19/30 Read block 588 len 8192 20/30 Write block 471 len 16384 data: 12 11 12 12 21/30 Read block 93 len 24576 22/30 Read block 283 len 24576 23/30 Read block 301 len 8192 24/30 Read block 560 len 24576 25/30 Flush 26/30 Write block 387 len 8192 data: 10 7 27/30 Write block 271 len 40960 data: 13 16 17 19 20 19 18 16 15 12 28/30 Write block 573 len 28672 data: 16 15 15 14 15 15 13 29/30 Write block 411 len 4096 data: 14 30/30 Read block 150 len 16384 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Sent Sent Done false 1028 Acked Write 1 Sent Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 0 6 Done 2 2 5 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:15.710395548Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1952} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 350 350 350 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 188 188 188 353 353 353 F F F 3 060-079 190 190 190 354 354 354 F F F 4 080-099 188 188 188 353 353 353 F F F 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 188 188 188 353 353 353 F F F 9 180-199 188 188 188 353 353 353 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 190 190 190 354 354 354 T T T 14 280-299 190 190 190 354 354 354 T T T 15 300-319 188 188 188 353 353 353 F F F 16 320-339 188 188 188 353 353 353 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 188 188 188 353 353 353 F F F 19 380-399 188 188 188 353 353 353 T T T 20 400-419 188 188 188 353 353 353 T T T 21 420-439 190 190 190 354 354 354 F F F 22 440-459 188 188 188 353 353 353 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 190 190 190 354 354 354 F F F 26 520-539 190 190 190 354 354 354 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 188 188 188 353 353 353 T T T 29 580-599 190 190 190 354 354 354 F F F Max gen: 190, Max flush: 354 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 350 350 350 F F F 1 020-039 184 184 184 347 347 347 F F F 2 040-059 188 188 188 353 353 353 F F F 3 060-079 190 190 190 354 354 354 F F F 4 080-099 188 188 188 353 353 353 F F F 5 100-119 184 184 184 345 345 345 F F F 6 120-139 184 184 184 347 347 347 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 188 188 188 353 353 353 F F F 9 180-199 188 188 188 353 353 353 F F F 10 200-219 180 180 180 339 339 339 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 191 191 191 355 355 355 F F F 15 300-319 188 188 188 353 353 353 F F F 16 320-339 188 188 188 353 353 353 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 188 188 188 353 353 353 F F F 19 380-399 191 191 191 355 355 355 F F F 20 400-419 191 191 191 355 355 355 F F F 21 420-439 190 190 190 354 354 354 F F F 22 440-459 188 188 188 353 353 353 F F F 23 460-479 190 190 190 354 354 354 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 190 190 190 354 354 354 F F F 26 520-539 190 190 190 354 354 354 F F F 27 540-559 188 188 188 353 353 353 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 190 190 190 354 354 354 F F F Max gen: 191, Max flush: 355 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:19.973494916Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:20.015365103Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.015738093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.046792195Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.047508646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.047544745Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.047558675Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.047572744Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} {"msg":"Crucible 045885f4-902e-45b6-a1bd-cb9aa05b08df has session id: f135fdbd-0c73-4e52-98cf-37f46279cc90","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048249947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Upstairs opts: Upstairs UUID: 045885f4-902e-45b6-a1bd-cb9aa05b08df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048288226Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Crucible stats registered with UUID: 045885f4-902e-45b6-a1bd-cb9aa05b08df","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048303975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:20.048316795Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048803672Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048826782Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} {"msg":"045885f4-902e-45b6-a1bd-cb9aa05b08df active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.04888886Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.04890614Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"0","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048940379Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"0","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048957058Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"1","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048972408Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"1","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.048985927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"2","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.049000797Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"2","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.049014167Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.049028366Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.049041676Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"ds_connection connected from Ok(127.0.0.1:37525)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.049168313Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"ds_connection connected from Ok(127.0.0.1:47201)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.049242711Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"ds_connection connected from Ok(127.0.0.1:42455)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.0492743Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.05002312Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"2","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.050062719Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"1","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.050106268Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"0","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.050821319Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"1","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.050952546Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"0","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051045963Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"2","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051327946Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051400574Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[0]R flush_numbers[0..12]: [350, 347, 353, 354, 353, 345, 347, 342, 353, 353, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051603288Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[0]R generation[0..12]: [186, 184, 188, 190, 188, 184, 184, 181, 188, 188, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051625248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051640388Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[1]R flush_numbers[0..12]: [350, 347, 353, 354, 353, 345, 347, 342, 353, 353, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051654367Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[1]R generation[0..12]: [186, 184, 188, 190, 188, 184, 184, 181, 188, 188, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051667947Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051681726Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[2]R flush_numbers[0..12]: [350, 347, 353, 354, 353, 345, 347, 342, 353, 353, 339, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051695296Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[2]R generation[0..12]: [186, 184, 188, 190, 188, 184, 184, 181, 188, 188, 180, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051708536Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051722075Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051737845Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051750965Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Next flush: 356","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051763724Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051784974Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"":"downstairs","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051797993Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051809733Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"0","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051822553Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"1","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051835202Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"client":"2","":"downstairs","session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051847972Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051859822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"045885f4-902e-45b6-a1bd-cb9aa05b08df is now active with session: f135fdbd-0c73-4e52-98cf-37f46279cc90","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051872361Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051883971Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960,"session_id":"f135fdbd-0c73-4e52-98cf-37f46279cc90"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.051895831Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} {"msg":"Activated sub_volume 045885f4-902e-45b6-a1bd-cb9aa05b08df","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:20.05190686Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 508 len 24576 data: 16 15 16 19 16 16 03/30 Write block 107 len 4096 data: 12 04/30 Write block 294 len 8192 data: 13 12 05/30 Write block 452 len 36864 data: 17 15 16 15 14 16 18 17 14 06/30 Read block 101 len 8192 07/30 Write block 127 len 28672 data: 13 13 14 10 10 12 12 08/30 Write block 199 len 8192 data: 9 9 09/30 Read block 291 len 24576 10/30 Read block 269 len 40960 11/30 Write block 500 len 8192 data: 9 9 12/30 Write block 204 len 4096 data: 9 13/30 Flush 14/30 Write block 413 len 32768 data: 15 12 13 15 15 17 18 18 15/30 Read block 108 len 32768 16/30 Write block 169 len 4096 data: 14 17/30 Read block 352 len 4096 18/30 Flush 19/30 Write block 542 len 4096 data: 17 20/30 Read block 496 len 32768 21/30 Read block 460 len 28672 22/30 Read block 18 len 12288 23/30 Read block 477 len 36864 24/30 Read block 420 len 24576 25/30 Write block 336 len 12288 data: 16 17 20 26/30 Read block 16 len 28672 27/30 Write block 545 len 16384 data: 18 16 22 21 28/30 Write block 18 len 20480 data: 17 16 14 13 13 29/30 Read block 406 len 36864 30/30 Read block 83 len 28672 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 12 11 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:20.073082172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1960} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 350 350 350 T T T 1 020-039 184 184 184 347 347 347 T T T 2 040-059 188 188 188 353 353 353 F F F 3 060-079 190 190 190 354 354 354 F F F 4 080-099 188 188 188 353 353 353 F F F 5 100-119 192 192 192 357 357 357 F F F 6 120-139 192 192 192 357 357 357 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 192 192 192 358 358 358 F F F 9 180-199 192 192 192 357 357 357 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 192 192 192 357 357 357 F F F 15 300-319 188 188 188 353 353 353 F F F 16 320-339 188 188 188 353 353 353 T T T 17 340-359 185 185 185 348 348 348 F F F 18 360-379 188 188 188 353 353 353 F F F 19 380-399 191 191 191 355 355 355 F F F 20 400-419 192 192 192 358 358 358 F F F 21 420-439 192 192 192 358 358 358 F F F 22 440-459 192 192 192 357 357 357 F F F 23 460-479 192 192 192 357 357 357 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 192 192 192 357 357 357 F F F 26 520-539 190 190 190 354 354 354 F F F 27 540-559 188 188 188 353 353 353 T T T 28 560-579 191 191 191 355 355 355 F F F 29 580-599 190 190 190 354 354 354 F F F Max gen: 192, Max flush: 358 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 188 188 188 353 353 353 F F F 3 060-079 190 190 190 354 354 354 F F F 4 080-099 188 188 188 353 353 353 F F F 5 100-119 192 192 192 357 357 357 F F F 6 120-139 192 192 192 357 357 357 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 192 192 192 358 358 358 F F F 9 180-199 192 192 192 357 357 357 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 184 184 184 346 346 346 F F F 12 240-259 185 185 185 348 348 348 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 192 192 192 357 357 357 F F F 15 300-319 188 188 188 353 353 353 F F F 16 320-339 193 193 193 359 359 359 F F F 17 340-359 185 185 185 348 348 348 F F F 18 360-379 188 188 188 353 353 353 F F F 19 380-399 191 191 191 355 355 355 F F F 20 400-419 192 192 192 358 358 358 F F F 21 420-439 192 192 192 358 358 358 F F F 22 440-459 192 192 192 357 357 357 F F F 23 460-479 192 192 192 357 357 357 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 192 192 192 357 357 357 F F F 26 520-539 190 190 190 354 354 354 F F F 27 540-559 193 193 193 359 359 359 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 190 190 190 354 354 354 F F F Max gen: 193, Max flush: 359 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.330395136Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:24.372444208Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.372840317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.403607317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.404280469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.404313418Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.404326828Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.404341048Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} {"msg":"Crucible 4d2a33c4-c737-4e5f-b8a2-74897f34f8c3 has session id: 03ca4b05-f85b-46a2-836b-5edd2c228827","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.404949582Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Upstairs opts: Upstairs UUID: 4d2a33c4-c737-4e5f-b8a2-74897f34f8c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.404994361Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Crucible stats registered with UUID: 4d2a33c4-c737-4e5f-b8a2-74897f34f8c3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.40502076Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:24.405043629Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405439189Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405471998Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} {"msg":"4d2a33c4-c737-4e5f-b8a2-74897f34f8c3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405527236Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405561406Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"0","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405589885Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"0","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405614664Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"1","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405640913Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"1","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405665613Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"2","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405693492Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"2","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405719431Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.40575576Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405790249Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"ds_connection connected from Ok(127.0.0.1:49869)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405820939Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"ds_connection connected from Ok(127.0.0.1:49124)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405846648Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"ds_connection connected from Ok(127.0.0.1:63965)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.405883847Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.406621827Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"1","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.406656247Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"2","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.406820822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"0","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.407352498Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"2","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.407475505Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"1","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.407495875Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"0","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.407929173Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.40806066Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[0]R flush_numbers[0..12]: [359, 359, 353, 354, 353, 357, 357, 342, 358, 357, 357, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408103719Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[0]R generation[0..12]: [193, 193, 188, 190, 188, 192, 192, 181, 192, 192, 192, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408119568Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408137888Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[1]R flush_numbers[0..12]: [359, 359, 353, 354, 353, 357, 357, 342, 358, 357, 357, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408152177Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[1]R generation[0..12]: [193, 193, 188, 190, 188, 192, 192, 181, 192, 192, 192, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408165967Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408179617Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[2]R flush_numbers[0..12]: [359, 359, 353, 354, 353, 357, 357, 342, 358, 357, 357, 346]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408192896Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[2]R generation[0..12]: [193, 193, 188, 190, 188, 192, 192, 181, 192, 192, 192, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408206176Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408219696Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408232155Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408244975Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Next flush: 360","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408257364Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408269834Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"":"downstairs","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408282564Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408301623Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"0","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408315663Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,Wait for a query_work_queue command to finish before sending IO "client":"1","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408337952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"client":"2","":"downstairs","session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408350752Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408362682Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"4d2a33c4-c737-4e5f-b8a2-74897f34f8c3 is now active with session: 03ca4b05-f85b-46a2-836b-5edd2c228827","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408375061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408386531Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968,"session_id":"03ca4b05-f85b-46a2-836b-5edd2c228827"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408398311Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} {"msg":"Activated sub_volume 4d2a33c4-c737-4e5f-b8a2-74897f34f8c3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:24.408409351Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 62 len 36864 data: 18 17 16 16 16 17 18 17 13 02/30 Flush 03/30 Write block 328 len 8192 data: 12 13 04/30 Write block 389 len 40960 data: 9 11 16 20 20 25 25 20 23 28 05/30 Flush 06/30 Write block 90 len 40960 data: 19 17 13 17 15 16 18 18 19 19 07/30 Write block 399 len 28672 data: 25 26 22 20 19 17 17 08/30 Flush 09/30 Write block 67 len 8192 data: 18 19 10/30 Write block 349 len 28672 data: 14 13 14 13 10 13 12 11/30 Write block 392 len 4096 data: 21 12/30 Write block 337 len 12288 data: 18 21 19 13/30 Write block 309 len 20480 data: 21 18 17 16 16 14/30 Write block 439 len 4096 data: 11 15/30 Read block 426 len 4096 16/30 Write block 391 len 12288 data: 17 22 21 17/30 Write block 237 len 40960 data: 15 14 11 13 15 12 14 11 14 15 18/30 Read block 292 len 36864 19/30 Read block 327 len 4096 20/30 Write block 374 len 36864 data: 19 17 16 19 22 23 22 18 12 21/30 Write block 444 len 36864 data: 8 10 11 11 10 9 14 14 18 22/30 Flush 23/30 Write block 521 len 8192 data: 11 13 24/30 Write block 298 len 16384 data: 12 12 11 11 25/30 Flush 26/30 Write block 239 len 20480 data: 12 14 16 13 15 27/30 Write block 126 len 28672 data: 8 14 14 15 11 11 13 28/30 Write block 361 len 40960 data: 18 20 20 16 15 17 19 17 13 16 29/30 Read block 186 len 8192 30/30 Read block 158 len 28672 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:24.427905677Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1968} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 188 188 188 353 353 353 F F F 3 060-079 194 194 194 363 363 363 F F F 4 080-099 194 194 194 362 362 362 F F F 5 100-119 192 192 192 357 357 357 F F F 6 120-139 192 192 192 357 357 357 T T T 7 140-159 181 181 181 342 342 342 F F F 8 160-179 192 192 192 358 358 358 F F F 9 180-199 192 192 192 357 357 357 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 194 194 194 363 363 363 T T T 12 240-259 194 194 194 363 363 363 T T T 13 260-279 191 191 191 355 355 355 F F F 14 280-299 194 194 194 364 364 364 F F F 15 300-319 194 194 194 364 364 364 F F F 16 320-339 194 194 194 363 363 363 F F F 17 340-359 194 194 194 363 363 363 F F F 18 360-379 194 194 194 363 363 363 T T T 19 380-399 194 194 194 363 363 363 F F F 20 400-419 194 194 194 362 362 362 F F F 21 420-439 194 194 194 363 363 363 F F F 22 440-459 194 194 194 363 363 363 F F F 23 460-479 192 192 192 357 357 357 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 192 192 192 357 357 357 F F F 26 520-539 194 194 194 364 364 364 F F F 27 540-559 193 193 193 359 359 359 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 190 190 190 354 354 354 F F F Max gen: 194, Max flush: 364 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 188 188 188 353 353 353 F F F 3 060-079 194 194 194 363 363 363 F F F 4 080-099 194 194 194 362 362 362 F F F 5 100-119 192 192 192 357 357 357 F F F 6 120-139 195 195 195 365 365 365 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 192 192 192 358 358 358 F F F 9 180-199 192 192 192 357 357 357 F F F 10 200-219 192 192 192 357 357 357 F F F 11 220-239 195 195 195 365 365 365 F F F 12 240-259 195 195 195 365 365 365 F F F 13 260-279 191 191 191 355 355 355 F F F 14 280-299 194 194 194 364 364 364 F F F 15 300-319 194 194 194 364 364 364 F F F 16 320-339 194 194 194 363 363 363 F F F 17 340-359 194 194 194 363 363 363 F F F 18 360-379 195 195 195 365 365 365 F F F 19 380-399 194 194 194 363 363 363 F F F 20 400-419 194 194 194 362 362 362 F F F 21 420-439 194 194 194 363 363 363 F F F 22 440-459 194 194 194 363 363 363 F F F 23 460-479 192 192 192 357 357 357 F F F 24 480-499 182 182 182 343 343 343 F F F 25 500-519 192 192 192 357 357 357 F F F 26 520-539 194 194 194 364 364 364 F F F 27 540-559 193 193 193 359 359 359 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 190 190 190 354 354 354 F F F Max gen: 195, Max flush: 365 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.684473019Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.728961927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.72996561Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.7300003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.730019469Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.730033848Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} {"msg":"Crucible 14670686-4dca-4d00-816c-7ee24e97dac4 has session id: f79ee9a5-968c-4825-9922-d4d3ec8e0eab","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.730708281Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Upstairs opts: Upstairs UUID: 14670686-4dca-4d00-816c-7ee24e97dac4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.730755669Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Crucible stats registered with UUID: 14670686-4dca-4d00-816c-7ee24e97dac4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.730780499Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:28.730802088Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731191988Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731214917Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} {"msg":"14670686-4dca-4d00-816c-7ee24e97dac4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731275506Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731292346Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"0","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731318405Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"0","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731332404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"1","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731346394Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"1","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731359394Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"2","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731382563Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"2","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731405903Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731430572Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731454111Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"ds_connection connected from Ok(127.0.0.1:46639)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731478561Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"ds_connection connected from Ok(127.0.0.1:51974)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731516829Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"ds_connection connected from Ok(127.0.0.1:35702)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.731543779Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.73225932Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"2","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.732303069Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"0","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.732987761Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"2","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.733136977Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"1","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.733164706Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"0","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.733614824Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.733691752Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.733709952Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"1","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[0]R flush_numbers[0..12]: [359, 359, 353, 363, 362, 357, 365, 342, 358, 357, 357, 365]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734247977Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[0]R generation[0..12]: [193, 193, 188, 194, 194, 192, 195, 181, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734268017Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734282247Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[1]R flush_numbers[0..12]: [359, 359, 353, 363, 362, 357, 365, 342, 358, 357, 357, 365]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734295786Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[1]R generation[0..12]: [193, 193, 188, 194, 194, 192, 195, 181, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734309156Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734322656Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[2]R flush_numbers[0..12]: [359, 359, 353, 363, 362, 357, 365, 342, 358, 357, 357, 365]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734336225Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[2]R generation[0..12]: [193, 193, 188, 194, 194, 192, 195, 181, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734349435Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734362994Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734378574Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734391754Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Next flush: 366","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734404353Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734417003Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"":"downstairs","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-19T17:14:28.734429713Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734444472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"0","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734457472Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"1","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734470162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"client":"2","":"downstairs","session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734483031Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734494801Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"14670686-4dca-4d00-816c-7ee24e97dac4 is now active with session: f79ee9a5-968c-4825-9922-d4d3ec8e0eab","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.734507191Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.73451871Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976,"session_id":"f79ee9a5-968c-4825-9922-d4d3ec8e0eab"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.73453061Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} {"msg":"Activated sub_volume 14670686-4dca-4d00-816c-7ee24e97dac4","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:28.73454143Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 32768 data: 20 20 24 25 23 22 20 18 02/30 Read block 93 len 24576 03/30 Write block 107 len 24576 data: 13 11 11 9 13 11 04/30 Write block 495 len 28672 data: 18 17 17 16 10 10 10 05/30 Write block 586 len 32768 data: 13 15 13 13 13 11 11 11 06/30 Read block 146 len 12288 07/30 Write block 512 len 4096 data: 17 08/30 Write block 27 len 28672 data: 17 19 19 17 17 18 18 09/30 Write block 47 len 36864 data: 14 11 11 12 12 12 12 14 15 10/30 Read block 459 len 36864 11/30 Write block 132 len 8192 data: 14 13 12/30 Write block 398 len 40960 data: 29 26 27 23 21 20 18 18 15 17 13/30 Read block 158 len 36864 14/30 Read block 104 len 20480 15/30 Read block 92 len 40960 16/30 Write block 28 len 16384 data: 20 20 18 18 17/30 Write block 499 len 8192 data: 11 11 18/30 Write block 369 len 16384 data: 14 17 16 16 19/30 Read block 552 len 20480 20/30 Write block 199 len 40960 data: 10 10 12 14 11 10 10 12 12 11 21/30 Read block 93 len 20480 22/30 Read block 345 len 16384 23/30 Read block 57 len 16384 24/30 Read block 140 len 20480 25/30 Write block 296 len 24576 data: 11 10 13 13 12 12 26/30 Read block 117 len 32768 27/30 Write block 264 len 36864 data: 14 14 16 17 17 17 19 14 17 28/30 Read block 451 len 16384 29/30 Write block 175 len 32768 data: 20 16 15 19 20 16 13 13 30/30 Write block 192 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:28.756262308Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1976} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 T T T 2 040-059 188 188 188 353 353 353 T T T 3 060-079 194 194 194 363 363 363 F F F 4 080-099 194 194 194 362 362 362 T T T 5 100-119 192 192 192 357 357 357 T T T 6 120-139 195 195 195 365 365 365 T T T 7 140-159 181 181 181 342 342 342 F F F 8 160-179 192 192 192 358 358 358 T T T 9 180-199 192 192 192 357 357 357 T T T 10 200-219 192 192 192 357 357 357 T T T 11 220-239 195 195 195 365 365 365 F F F 12 240-259 195 195 195 365 365 365 F F F 13 260-279 191 191 191 355 355 355 T T T 14 280-299 194 194 194 364 364 364 T T T 15 300-319 194 194 194 364 364 364 T T T 16 320-339 194 194 194 363 363 363 F F F 17 340-359 194 194 194 363 363 363 F F F 18 360-379 195 195 195 365 365 365 T T T 19 380-399 194 194 194 363 363 363 T T T 20 400-419 194 194 194 362 362 362 T T T 21 420-439 194 194 194 363 363 363 F F F 22 440-459 194 194 194 363 363 363 F F F 23 460-479 192 192 192 357 357 357 F F F 24 480-499 182 182 182 343 343 343 T T T 25 500-519 192 192 192 357 357 357 T T T 26 520-539 194 194 194 364 364 364 F F F 27 540-559 193 193 193 359 359 359 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 190 190 190 354 354 354 T T T Max gen: 195, Max flush: 365 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 196 196 196 366 366 366 F F F 2 040-059 196 196 196 366 366 366 F F F 3 060-079 194 194 194 363 363 363 F F F 4 080-099 196 196 196 366 366 366 F F F 5 100-119 196 196 196 366 366 366 F F F 6 120-139 196 196 196 366 366 366 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 196 196 196 366 366 366 F F F 9 180-199 196 196 196 366 366 366 F F F 10 200-219 196 196 196 366 366 366 F F F 11 220-239 195 195 195 365 365 365 F F F 12 240-259 195 195 195 365 365 365 F F F 13 260-279 196 196 196 366 366 366 F F F 14 280-299 196 196 196 366 366 366 F F F 15 300-319 196 196 196 366 366 366 F F F 16 320-339 194 194 194 363 363 363 F F F 17 340-359 194 194 194 363 363 363 F F F 18 360-379 196 196 196 366 366 366 F F F 19 380-399 196 196 196 366 366 366 F F F 20 400-419 196 196 196 366 366 366 F F F 21 420-439 194 194 194 363 363 363 F F F 22 440-459 194 194 194 363 363 363 F F F 23 460-479 192 192 192 357 357 357 F F F 24 480-499 196 196 196 366 366 366 F F F 25 500-519 196 196 196 366 366 366 F F F 26 520-539 194 194 194 364 364 364 F F F 27 540-559 193 193 193 359 359 359 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 196 196 196 366 366 366 F F F Max gen: 196, Max flush: 366 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.058658093Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:43.100587289Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.100987688Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.13166045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.132237215Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"f97cec98046223ddb58af06ce2781e9bf87fb3e2\",\n git_commit_timestamp: \"2025-03-19T16:40:31.000000000Z\",\n git_branch: \"iliana/update-omicron\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.132271934Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.132285684Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.132299973Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} {"msg":"Crucible 93bb315b-b3dd-48e5-a87b-9efc614d91a1 has session id: 268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.132942986Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Upstairs opts: Upstairs UUID: 93bb315b-b3dd-48e5-a87b-9efc614d91a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.132983125Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Crucible stats registered with UUID: 93bb315b-b3dd-48e5-a87b-9efc614d91a1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133015175Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:43.133029144Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133375385Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133398714Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} {"msg":"93bb315b-b3dd-48e5-a87b-9efc614d91a1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133458183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133475272Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"0","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133498942Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"0","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133521751Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"1","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.13355443Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"1","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.13358322Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"2","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133611899Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"2","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133636248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133662787Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133688767Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56957)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133713306Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56089)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133737205Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33941)","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.133760565Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.134427927Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"1","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.134457317Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"2","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.134589183Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"0","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135153468Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"1","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135259355Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"0","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135315504Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"2","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135660775Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135761402Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[0]R flush_numbers[0..12]: [359, 366, 366, 363, 366, 366, 366, 342, 366, 366, 366, 365]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135888249Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[0]R generation[0..12]: [193, 196, 196, 194, 196, 196, 196, 181, 196, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135911248Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135926508Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[1]R flush_numbers[0..12]: [359, 366, 366, 363, 366, 366, 366, 342, 366, 366, 366, 365]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135940478Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[1]R generation[0..12]: [193, 196, 196, 194, 196, 196, 196, 181, 196, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135954737Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135968477Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[2]R flush_numbers[0..12]: [359, 366, 366, 363, 366, 366, 366, 342, 366, 366, 366, 365]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135981656Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[2]R generation[0..12]: [193, 196, 196, 194, 196, 196, 196, 181, 196, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.135994936Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136008646Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Max found gen is 197","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136021215Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Generation requested: 198 >= found:197","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136034045Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Next flush: 367","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136051155Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136064174Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"":"downstairs","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136083404Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136095213Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"0","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136108103Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"1","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136120823Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"client":"2","":"downstairs","session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136133792Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136145822Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"93bb315b-b3dd-48e5-a87b-9efc614d91a1 is now active with session: 268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136158172Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136169901Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984,"session_id":"268aeaf0-fcb1-484e-ace2-f9b82ec8ae6c"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136181931Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} {"msg":"Activated sub_volume 93bb315b-b3dd-48e5-a87b-9efc614d91a1","v":0,"name":"crucible","level":30,"time":"2025-03-19T17:14:43.136192791Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 355 len 24576 data: 13 12 11 13 17 17 02/30 Read block 487 len 20480 03/30 Read block 546 len 36864 04/30 Read block 504 len 32768 05/30 Write block 33 len 4096 data: 19 06/30 Flush 07/30 Write block 116 len 36864 data: 18 18 16 15 13 11 10 10 10 08/30 Write block 593 len 24576 data: 12 11 10 12 10 7 09/30 Read block 133 len 32768 10/30 Write block 247 len 8192 data: 19 19 11/30 Flush 12/30 Write block 235 len 36864 data: 14 14 16 15 13 15 17 14 16 13/30 Write block 175 len 4096 data: 21 14/30 Write block 494 len 12288 data: 18 19 18 15/30 Write block 271 len 4096 data: 15 16/30 Write block 260 len 12288 data: 14 19 18 17/30 Flush 18/30 Flush 19/30 Write block 27 len 40960 data: 18 21 21 19 19 19 20 16 18 17 20/30 Write block 168 len 32768 data: 14 15 16 18 15 16 17 22 21/30 Write block 455 len 8192 data: 16 15 22/30 Write block 234 len 28672 data: 9 15 15 17 16 14 16 23/30 Read block 204 len 32768 24/30 Read block 401 len 12288 25/30 Write block 91 len 36864 data: 18 14 18 16 17 19 19 21 21 26/30 Write block 392 len 24576 data: 23 22 26 26 21 24 27/30 Write block 229 len 8192 data: 11 10 28/30 Write block 407 len 28672 data: 18 18 15 16 15 14 16 29/30 Read block 233 len 40960 30/30 Read block 571 len 12288 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-19T17:14:43.154750162Z","hostname":"w-01JPQNADMP9B2VH13EXY31CFA6","pid":1984} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 198 198 198 367 367 367 T T T 2 040-059 196 196 196 366 366 366 F F F 3 060-079 194 194 194 363 363 363 F F F 4 080-099 196 196 196 366 366 366 T T T 5 100-119 198 198 198 368 368 368 F F F 6 120-139 198 198 198 368 368 368 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 198 198 198 369 369 369 T T T 9 180-199 196 196 196 366 366 366 F F F 10 200-219 196 196 196 366 366 366 F F F 11 220-239 198 198 198 369 369 369 T T T 12 240-259 198 198 198 369 369 369 T T T 13 260-279 198 198 198 369 369 369 F F F 14 280-299 196 196 196 366 366 366 F F F 15 300-319 196 196 196 366 366 366 F F F 16 320-339 194 194 194 363 363 363 F F F 17 340-359 198 198 198 367 367 367 F F F 18 360-379 198 198 198 367 367 367 F F F 19 380-399 196 196 196 366 366 366 T T T 20 400-419 196 196 196 366 366 366 T T T 21 420-439 194 194 194 363 363 363 F F F 22 440-459 194 194 194 363 363 363 T T T 23 460-479 192 192 192 357 357 357 F F F 24 480-499 198 198 198 369 369 369 F F F 25 500-519 196 196 196 366 366 366 F F F 26 520-539 194 194 194 364 364 364 F F F 27 540-559 193 193 193 359 359 359 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 198 198 198 368 368 368 F F F Max gen: 198, Max flush: 369 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 199 199 199 370 370 370 F F F 2 040-059 196 196 196 366 366 366 F F F 3 060-079 194 194 194 363 363 363 F F F 4 080-099 199 199 199 370 370 370 F F F 5 100-119 198 198 198 368 368 368 F F F 6 120-139 198 198 198 368 368 368 F F F 7 140-159 181 181 181 342 342 342 F F F 8 160-179 199 199 199 370 370 370 F F F 9 180-199 196 196 196 366 366 366 F F F 10 200-219 196 196 196 366 366 366 F F F 11 220-239 199 199 199 370 370 370 F F F 12 240-259 199 199 199 370 370 370 F F F 13 260-279 198 198 198 369 369 369 F F F 14 280-299 196 196 196 366 366 366 F F F 15 300-319 196 196 196 366 366 366 F F F 16 320-339 194 194 194 363 363 363 F F F 17 340-359 198 198 198 367 367 367 F F F 18 360-379 198 198 198 367 367 367 F F F 19 380-399 199 199 199 370 370 370 F F F 20 400-419 199 199 199 370 370 370 F F F 21 420-439 194 194 194 363 363 363 F F F 22 440-459 199 199 199 370 370 370 F F F 23 460-479 192 192 192 357 357 357 F F F 24 480-499 198 198 198 369 369 369 F F F 25 500-519 196 196 196 366 366 366 F F F 26 520-539 194 194 194 364 364 364 F F F 27 540-559 193 193 193 359 359 359 F F F 28 560-579 191 191 191 355 355 355 F F F 29 580-599 198 198 198 368 368 368 F F F Max gen: 199, Max flush: 370 17:44 Test duration Test completed real 17:44.376022334 user 50.091109420 sys 19.241205458 trap 0.066059299 tflt 0.081094549 dflt 0.132276774 kflt 0.006111364 lock 27:53:07.577328752 slp 1:26:37.956195722 lat 5.914553247 stop 0.064099460