Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.3056253 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.020134732 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.023019753 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1193 1194 1195 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.465021377Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.519552147Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.520772969Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.520868516Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.520888253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.520903262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} {"msg":"Crucible fd8ae82e-1c90-425a-a07b-4540933e18f7 has session id: d89bc4a1-51b3-4791-9296-30961ff41408","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.521835813Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Upstairs opts: Upstairs UUID: fd8ae82e-1c90-425a-a07b-4540933e18f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.521909343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Crucible stats registered with UUID: fd8ae82e-1c90-425a-a07b-4540933e18f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.52192916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:33.521944118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.522881759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.522980466Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} {"msg":"fd8ae82e-1c90-425a-a07b-4540933e18f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523041177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523060325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"0","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523151722Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"0","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.52317093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"1","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523188577Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"1","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523212254Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"2","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.52323963Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"2","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523351135Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523379571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523407817Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"ds_connection connected from Ok(127.0.0.1:35691)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523592222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"ds_connection connected from Ok(127.0.0.1:35934)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523627007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"ds_connection connected from Ok(127.0.0.1:63857)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.523653183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.524960943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"0","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.524997508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"2","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.525033323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"1","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.526244697Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"0","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.526287721Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"1","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.526380698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"2","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.52702138Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527047097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527135534Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527156102Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527171399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527185398Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527199456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527214063Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527228412Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527242839Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527260197Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527273755Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527287124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527300352Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.52731358Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"":"downstairs","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527327148Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527345105Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527373172Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"1","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527392219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"client":"2","":"downstairs","session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527416186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527431284Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"fd8ae82e-1c90-425a-a07b-4540933e18f7 is now active with session: d89bc4a1-51b3-4791-9296-30961ff41408","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527444452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.52745662Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198,"session_id":"d89bc4a1-51b3-4791-9296-30961ff41408"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527469048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} {"msg":"Activated sub_volume fd8ae82e-1c90-425a-a07b-4540933e18f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:33.527480447Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 256 Done Done Done false 1005 Acked Read 88 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:37.618085668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1198} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.65137256Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:37.695165156Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.695643781Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.727778911Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.728480204Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.728524068Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.728539286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.728554024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} {"msg":"Crucible 71aeb03f-f37c-4789-94bf-d98792e5185d has session id: e53a9c05-8d41-4670-8900-e3d361f9c8ee","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729208904Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Upstairs opts: Upstairs UUID: 71aeb03f-f37c-4789-94bf-d98792e5185d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729245459Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Crucible stats registered with UUID: 71aeb03f-f37c-4789-94bf-d98792e5185d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729266456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:37.729280154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729625437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729648993Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} {"msg":"71aeb03f-f37c-4789-94bf-d98792e5185d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729702236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729729353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"0","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729767777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"0","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729795393Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"1","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.72982245Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"1","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729849846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"2","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729879372Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"2","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729908438Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729934984Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729960791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:61591)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.729987677Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:57264)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.730076395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:52109)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.730107001Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.730535722Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"2","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.730655095Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"1","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.73076945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"0","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.731177673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"2","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.731211479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"1","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.731356309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"0","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.731840592Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.731878887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.731986532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732007249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732022827Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732037055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732054453Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732069161Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732082959Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732096767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732111225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732125523Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732138891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732152239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732172246Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"":"downstairs","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732186195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732198603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"0","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732212351Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"1","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732226139Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"client":"2","":"downstairs","session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732239687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732252186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"71aeb03f-f37c-4789-94bf-d98792e5185d is now active with session: e53a9c05-8d41-4670-8900-e3d361f9c8ee","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732267803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732295789Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200,"session_id":"e53a9c05-8d41-4670-8900-e3d361f9c8ee"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732309708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} {"msg":"Activated sub_volume 71aeb03f-f37c-4789-94bf-d98792e5185d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:37.732321336Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 566 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 360 len 28672 03/30 Flush 04/30 Write block 59 len 16384 data: 2 2 2 2 05/30 Read block 135 len 8192 06/30 Flush 07/30 Write block 180 len 4096 data: 2 08/30 Read block 213 len 4096 09/30 Flush 10/30 Write block 221 len 28672 data: 2 2 2 2 2 2 2 11/30 Read block 509 len 36864 12/30 Write block 259 len 20480 data: 2 2 2 2 2 13/30 Write block 217 len 28672 data: 2 2 2 2 3 3 3 14/30 Write block 169 len 20480 data: 2 2 2 2 2 15/30 Write block 363 len 32768 data: 2 2 2 2 2 2 2 2 16/30 Read block 192 len 40960 17/30 Read block 164 len 36864 18/30 Read block 188 len 8192 19/30 Read block 0 len 24576 20/30 Write block 254 len 36864 data: 2 2 2 2 2 3 3 3 3 21/30 Read block 236 len 32768 22/30 Read block 346 len 16384 23/30 Write block 549 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Read block 428 len 16384 25/30 Read block 100 len 8192 26/30 Read block 58 len 24576 27/30 Write block 559 len 28672 data: 2 2 2 2 2 2 2 28/30 Read block 496 len 4096 29/30 Read block 424 len 8192 30/30 Read block 192 len 28672 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 7 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 20 21 20 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:37.753951271Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1200} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 T T T 9 180-199 2 2 2 4 4 4 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 1 1 1 1 1 1 T T T 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 2 2 2 2 2 2 T T T 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 2 2 2 4 4 4 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 3 3 3 5 5 5 F F F 12 240-259 3 3 3 5 5 5 F F F 13 260-279 3 3 3 5 5 5 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 5 5 5 F F F 28 560-579 3 3 3 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.04838238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:42.091413181Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.091833713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.123758612Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.124688684Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.124735448Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.124753575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.124771743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} {"msg":"Crucible 826918e1-f162-4ded-a907-b6d63527556d has session id: 15da76fb-1f29-45d1-8012-4948cbd1f6a0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.125379359Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Upstairs opts: Upstairs UUID: 826918e1-f162-4ded-a907-b6d63527556d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.125429402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Crucible stats registered with UUID: 826918e1-f162-4ded-a907-b6d63527556d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.12544998Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:42.125466417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.125911456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.125944321Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} {"msg":"826918e1-f162-4ded-a907-b6d63527556d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.125999814Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.12602692Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"0","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126060556Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"0","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126075554Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"1","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126090811Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"1","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126104909Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"2","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126119837Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"2","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126133575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126148444Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126162891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63014)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126267007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58744)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126305292Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53396)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.126408968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.127074816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"1","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.127113211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"2","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.127210587Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"0","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.127690351Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"1","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.127731306Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"2","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.127863248Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"0","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128338322Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128359709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 3, 3, 1, 1, 1, 1, 5, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128472864Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[0]R generation[0..12]: [1, 1, 2, 2, 1, 1, 1, 1, 3, 2, 3, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128492521Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128509069Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 3, 3, 1, 1, 1, 1, 5, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128523647Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[1]R generation[0..12]: [1, 1, 2, 2, 1, 1, 1, 1, 3, 2, 3, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128537605Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128552133Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 3, 3, 1, 1, 1, 1, 5, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128566031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[2]R generation[0..12]: [1, 1, 2, 2, 1, 1, 1, 1, 3, 2, 3, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128579899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128594317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128610815Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128625993Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128639491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128659318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"":"downstairs","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128673326Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128685924Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"0","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128699713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"1","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128713241Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"client":"2","":"downstairs","session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128726759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128739177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"826918e1-f162-4ded-a907-b6d63527556d is now active with session: 15da76fb-1f29-45d1-8012-4948cbd1f6a0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128752185Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128764384Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208,"session_id":"15da76fb-1f29-45d1-8012-4948cbd1f6a0"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.128777002Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} {"msg":"Activated sub_volume 826918e1-f162-4ded-a907-b6d63527556d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.12878854Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 560 len 40960 data: 3 3 3 3 3 3 3 3 3 3 02/30 Write block 584 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 159 len 8192 04/30 Flush 05/30 Write block 562 len 4096 data: 4 06/30 Write block 39 len 16384 data: 2 2 2 2 07/30 Read block 173 len 32768 08/30 Read block 317 len 4096 09/30 Read block 121 len 40960 10/30 Write block 588 len 4096 data: 3 11/30 Read block 294 len 40960 12/30 Read block 358 len 24576 13/30 Write block 226 len 4096 data: 3 14/30 Read block 546 len 32768 15/30 Read block 123 len 20480 16/30 Read block 320 len 12288 17/30 Read block 574 len 16384 18/30 Flush 19/30 Write block 3 len 4096 data: 2 20/30 Read block 278 len 36864 21/30 Read block 539 len 32768 22/30 Flush 23/30 Write block 482 len 36864 data: 2 2 2 2 2 2 2 2 2 24/30 Read block 419 len 40960 25/30 Read block 140 len 8192 26/30 Write block 395 len 40960 data: 2 2 2 2 2 2 2 2 2 2 27/30 Write block 564 len 20480 data: 4 4 4 4 4 28/30 Read block 65 len 12288 29/30 Read block 530 len 12288 30/30 Read block 141 len 8192 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 7 8 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:42.149028866Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 2 2 2 4 4 4 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 3 3 3 5 5 5 F F F 13 260-279 3 3 3 5 5 5 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 T T T 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 5 5 5 F F F 28 560-579 4 4 4 7 7 7 T T T 29 580-599 4 4 4 7 7 7 F F F Max gen: 4, Max flush: 8 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 2 2 2 4 4 4 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 3 3 3 5 5 5 F F F 13 260-279 3 3 3 5 5 5 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 5 5 5 9 9 9 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 5 5 5 9 9 9 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 3 3 3 5 5 5 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 4 4 4 7 7 7 F F F Max gen: 5, Max flush: 9 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.413398889Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.458134826Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.458941895Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.45897744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.458992088Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.459006706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} {"msg":"Crucible f95877ca-34f4-41e1-a17e-598059cbb776 has session id: ab59067b-ef5c-4939-8a02-4e79556e3108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.459518335Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Upstairs opts: Upstairs UUID: f95877ca-34f4-41e1-a17e-598059cbb776, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.459564329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Crucible stats registered with UUID: f95877ca-34f4-41e1-a17e-598059cbb776","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.459586216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:46.459599974Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.46013677Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460162717Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} {"msg":"f95877ca-34f4-41e1-a17e-598059cbb776 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460229768Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460246795Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"0","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460276511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"0","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460291659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"1","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460306527Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"1","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460328294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"2","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460397914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"2","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460441179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.46050617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460553063Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"ds_connection connected from Ok(127.0.0.1:62990)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460607406Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"ds_connection connected from Ok(127.0.0.1:54113)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460645431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"ds_connection connected from Ok(127.0.0.1:43756)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.460835064Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.461502882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"2","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.461532249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"0","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.46217518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"0","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.462337028Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"1","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.462369443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"2","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.462819021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.46297618Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463139617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"1","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[0]R flush_numbers[0..12]: [8, 7, 7, 3, 1, 1, 1, 1, 5, 4, 5, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463658296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 2, 1, 1, 1, 1, 3, 2, 3, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463678563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463693541Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[1]R flush_numbers[0..12]: [8, 7, 7, 3, 1, 1, 1, 1, 5, 4, 5, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463707879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 2, 1, 1, 1, 1, 3, 2, 3, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463722097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463740885Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[2]R flush_numbers[0..12]: [8, 7, 7, 3, 1, 1, 1, 1, 5, 4, 5, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463755723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 2, 1, 1, 1, 1, 3, 2, 3, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463769561Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463783839Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463797267Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463810955Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463833162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.46384653Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"":"downstairs","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463860328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463872747Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"0","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463886555Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"1","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463900153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"client":"2","":"downstairs","session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463913751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463926189Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"f95877ca-34f4-41e1-a17e-598059cbb776 is now active with session: ab59067b-ef5c-4939-8a02-4e79556e3108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463939307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463953575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216,"session_id":"ab59067b-ef5c-4939-8a02-4e79556e3108"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463966214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} {"msg":"Activated sub_volume f95877ca-34f4-41e1-a17e-598059cbb776","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:46.463977762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 237 len 4096 data: 2 02/30 Write block 116 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 548 len 8192 04/30 Read block 246 len 28672 05/30 Read block 221 len 4096 06/30 Write block 547 len 36864 data: 2 2 3 3 3 3 3 3 3 07/30 Read block 20 len 36864 08/30 Read block 142 len 4096 09/30 Read block 2 len 4096 10/30 Read block 41 len 36864 11/30 Write block 224 len 40960 data: 3 3 4 3 2 2 2 2 2 2 12/30 Read block 333 len 36864 13/30 Write block 441 len 12288 data: 2 2 2 14/30 Read block 398 len 12288 15/30 Read block 188 len 40960 16/30 Write block 594 len 4096 data: 2 17/30 Read block 433 len 16384 18/30 Read block 522 len 16384 19/30 Write block 154 len 20480 data: 2 2 2 2 2 20/30 Write block 521 len 36864 data: 2 2 2 2 2 2 2 2 2 21/30 Write block 425 len 16384 data: 2 2 2 2 22/30 Write block 252 len 28672 data: 2 2 3 3 3 3 3 23/30 Write block 108 len 12288 data: 2 2 2 24/30 Read block 123 len 20480 25/30 Write block 281 len 28672 data: 2 2 2 2 2 2 2 26/30 Write block 474 len 24576 data: 2 2 2 2 2 2 27/30 Read block 448 len 4096 28/30 Read block 302 len 16384 29/30 Read block 520 len 24576 30/30 Read block 468 len 28672 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:46.484217768Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 3 3 3 5 5 5 F F F 9 180-199 2 2 2 4 4 4 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 4 4 4 7 7 7 T T T 12 240-259 3 3 3 5 5 5 T T T 13 260-279 3 3 3 5 5 5 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 5 5 5 9 9 9 F F F 21 420-439 1 1 1 1 1 1 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 T T T 24 480-499 5 5 5 9 9 9 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 T T T 27 540-559 3 3 3 5 5 5 T T T 28 560-579 5 5 5 9 9 9 F F F 29 580-599 4 4 4 7 7 7 T T T Max gen: 5, Max flush: 9 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 6 6 6 10 10 10 F F F 6 120-139 6 6 6 10 10 10 F F F 7 140-159 6 6 6 10 10 10 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 2 2 2 4 4 4 F F F 10 200-219 3 3 3 5 5 5 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 6 6 6 10 10 10 F F F 13 260-279 3 3 3 5 5 5 F F F 14 280-299 6 6 6 10 10 10 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 5 5 5 9 9 9 F F F 21 420-439 6 6 6 10 10 10 F F F 22 440-459 6 6 6 10 10 10 F F F 23 460-479 6 6 6 10 10 10 F F F 24 480-499 5 5 5 9 9 9 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 6 6 6 10 10 10 F F F 27 540-559 6 6 6 10 10 10 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 6, Max flush: 10 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.769036392Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.813491617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.814274509Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.814309484Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.814323452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.81433816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} {"msg":"Crucible ecf81aec-2435-4e09-8d76-70c44db3f37b has session id: 7bbbcc91-0460-4c69-90a2-3beb07b08013","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.814911751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Upstairs opts: Upstairs UUID: ecf81aec-2435-4e09-8d76-70c44db3f37b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.814950166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Crucible stats registered with UUID: ecf81aec-2435-4e09-8d76-70c44db3f37b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.814965674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:00.814978552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815456847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815480223Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} {"msg":"ecf81aec-2435-4e09-8d76-70c44db3f37b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815553273Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815572041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"0","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815616545Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"0","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.81564947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"1","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815679806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"1","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815702713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"2","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815725939Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"2","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815750366Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815779572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815806718Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"ds_connection connected from Ok(127.0.0.1:63606)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.815836134Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"ds_connection connected from Ok(127.0.0.1:59735)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.81586539Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"ds_connection connected from Ok(127.0.0.1:58396)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.816026508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.81644769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"1","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.816487595Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"0","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.817162452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"0","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.817272887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"1","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.8178302Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.817863005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"2","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.817880183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.818478821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"2","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[0]R flush_numbers[0..12]: [8, 7, 7, 3, 1, 10, 10, 10, 5, 4, 5, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819006368Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 2, 1, 6, 6, 6, 3, 2, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819029865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819045673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[1]R flush_numbers[0..12]: [8, 7, 7, 3, 1, 10, 10, 10, 5, 4, 5, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819060481Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 2, 1, 6, 6, 6, 3, 2, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819074709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819089437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[2]R flush_numbers[0..12]: [8, 7, 7, 3, 1, 10, 10, 10, 5, 4, 5, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819103815Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 2, 1, 6, 6, 6, 3, 2, 3, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819117793Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819132201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Max found gen is 7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819145719Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Generation requested: 8 >= found:7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819164686Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:00.819180044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819201931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"":"downstairs","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819216309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819228978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"0","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819242916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"1","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819256834Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"client":"2","":"downstairs","session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819270732Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.81928333Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"ecf81aec-2435-4e09-8d76-70c44db3f37b is now active with session: 7bbbcc91-0460-4c69-90a2-3beb07b08013","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819296588Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819308957Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224,"session_id":"7bbbcc91-0460-4c69-90a2-3beb07b08013"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819321685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} {"msg":"Activated sub_volume ecf81aec-2435-4e09-8d76-70c44db3f37b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.819333343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 4096 data: 2 02/30 Read block 208 len 8192 03/30 Write block 421 len 16384 data: 2 2 2 2 04/30 Read block 134 len 4096 05/30 Write block 106 len 24576 data: 2 2 3 3 3 2 06/30 Read block 5 len 20480 07/30 Flush 08/30 Write block 552 len 16384 data: 4 4 4 4 09/30 Write block 139 len 16384 data: 2 2 2 2 10/30 Write block 497 len 12288 data: 2 2 2 11/30 Read block 544 len 12288 12/30 Read block 184 len 24576 13/30 Write block 291 len 8192 data: 2 2 14/30 Write block 189 len 24576 data: 2 2 2 2 2 2 15/30 Read block 404 len 16384 16/30 Read block 529 len 4096 17/30 Read block 107 len 12288 18/30 Write block 514 len 8192 data: 2 2 19/30 Read block 243 len 36864 20/30 Write block 69 len 8192 data: 2 2 21/30 Read block 292 len 32768 22/30 Read block 22 len 24576 23/30 Write block 361 len 20480 data: 2 2 3 3 3 24/30 Write block 369 len 36864 data: 3 3 2 2 2 2 2 2 2 25/30 Read block 400 len 20480 26/30 Write block 119 len 16384 data: 3 3 3 3 27/30 Write block 203 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Read block 6 len 32768 29/30 Read block 99 len 4096 30/30 Write block 416 len 20480 data: 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 4 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:00.838195779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 2 2 2 3 3 3 T T T 4 080-099 8 8 8 11 11 11 F F F 5 100-119 8 8 8 11 11 11 T T T 6 120-139 6 6 6 10 10 10 T T T 7 140-159 6 6 6 10 10 10 T T T 8 160-179 3 3 3 5 5 5 F F F 9 180-199 2 2 2 4 4 4 T T T 10 200-219 3 3 3 5 5 5 T T T 11 220-239 6 6 6 10 10 10 F F F 12 240-259 6 6 6 10 10 10 F F F 13 260-279 3 3 3 5 5 5 F F F 14 280-299 6 6 6 10 10 10 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 5 5 5 T T T 19 380-399 5 5 5 9 9 9 F F F 20 400-419 5 5 5 9 9 9 T T T 21 420-439 8 8 8 11 11 11 T T T 22 440-459 6 6 6 10 10 10 F F F 23 460-479 6 6 6 10 10 10 F F F 24 480-499 5 5 5 9 9 9 T T T 25 500-519 1 1 1 1 1 1 T T T 26 520-539 6 6 6 10 10 10 F F F 27 540-559 6 6 6 10 10 10 T T T 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 8, Max flush: 11 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 9 9 9 12 12 12 F F F 4 080-099 8 8 8 11 11 11 F F F 5 100-119 9 9 9 12 12 12 F F F 6 120-139 9 9 9 12 12 12 F F F 7 140-159 9 9 9 12 12 12 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 9 9 9 12 12 12 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 6 6 6 10 10 10 F F F 13 260-279 3 3 3 5 5 5 F F F 14 280-299 9 9 9 12 12 12 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 9 9 9 12 12 12 F F F 19 380-399 5 5 5 9 9 9 F F F 20 400-419 9 9 9 12 12 12 F F F 21 420-439 9 9 9 12 12 12 F F F 22 440-459 6 6 6 10 10 10 F F F 23 460-479 6 6 6 10 10 10 F F F 24 480-499 9 9 9 12 12 12 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 6 6 6 10 10 10 F F F 27 540-559 9 9 9 12 12 12 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 9, Max flush: 12 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.128088977Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:15.171272957Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.171750671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.20331219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.20396678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.204004805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.204022153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.20404475Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} {"msg":"Crucible 381635b8-aadb-49fe-86de-a94e122b409f has session id: 07457617-f36d-4379-9c58-f186a28afb9b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.204600873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Upstairs opts: Upstairs UUID: 381635b8-aadb-49fe-86de-a94e122b409f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.204640008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Crucible stats registered with UUID: 381635b8-aadb-49fe-86de-a94e122b409f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.204658995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:15.204675173Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205116402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205143479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} {"msg":"381635b8-aadb-49fe-86de-a94e122b409f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.20520497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205222968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"0","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205252863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"0","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205267701Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"1","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.20528287Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"1","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205297847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"2","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205323024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"2","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205363628Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205391644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205416721Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:51450)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205442357Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35048)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205467174Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61343)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.205493241Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.206229859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"1","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.206259755Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"2","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.206356982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"0","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.206955549Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"1","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.20702081Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"2","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207048567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"0","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207627677Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207740961Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[0]R flush_numbers[0..12]: [8, 7, 7, 12, 11, 12, 12, 12, 5, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207792644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 9, 8, 9, 9, 9, 3, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207815861Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207831669Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[1]R flush_numbers[0..12]: [8, 7, 7, 12, 11, 12, 12, 12, 5, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207846077Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 9, 8, 9, 9, 9, 3, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207860075Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207874603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[2]R flush_numbers[0..12]: [8, 7, 7, 12, 11, 12, 12, 12, 5, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207888631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 9, 8, 9, 9, 9, 3, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207902929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207917447Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207931095Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207944603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207957931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.2079711Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"":"downstairs","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207984698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.207997046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"0","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.208010824Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"1","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.208026462Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"client":"2","":"downstairs","session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.20804047Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.208053098Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"381635b8-aadb-49fe-86de-a94e122b409f is now active with session: 07457617-f36d-4379-9c58-f186a28afb9b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.208066317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id"Wait for a query_work_queue command to finish before sending IO :"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.208086944Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232,"session_id":"07457617-f36d-4379-9c58-f186a28afb9b"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.208099472Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} {"msg":"Activated sub_volume 381635b8-aadb-49fe-86de-a94e122b409f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:15.208110921Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 472 len 16384 data: 2 2 3 3 02/30 Flush 03/30 Write block 334 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Read block 116 len 32768 05/30 Flush 06/30 Write block 65 len 20480 data: 2 2 2 2 3 07/30 Write block 348 len 28672 data: 2 2 2 2 2 2 2 08/30 Read block 117 len 8192 09/30 Read block 455 len 20480 10/30 Flush 11/30 Write block 99 len 8192 data: 2 2 12/30 Read block 86 len 8192 13/30 Read block 540 len 40960 14/30 Write block 278 len 12288 data: 2 2 2 15/30 Flush 16/30 Write block 103 len 24576 data: 2 2 2 3 3 4 17/30 Flush 18/30 Write block 394 len 4096 data: 2 19/30 Read block 226 len 32768 20/30 Flush 21/30 Write block 525 len 4096 data: 3 22/30 Write block 330 len 36864 data: 2 2 2 2 3 3 3 3 3 23/30 Read block 269 len 4096 24/30 Read block 537 len 24576 25/30 Read block 226 len 4096 26/30 Read block 485 len 32768 27/30 Read block 172 len 12288 28/30 Read block 471 len 16384 29/30 Write block 93 len 36864 data: 2 2 2 2 2 2 3 3 2 30/30 Write block 243 len 20480 data: 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:15.227902208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 10 10 10 15 15 15 F F F 4 080-099 10 10 10 16 16 16 T T T 5 100-119 10 10 10 17 17 17 T T T 6 120-139 9 9 9 12 12 12 F F F 7 140-159 9 9 9 12 12 12 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 9 9 9 12 12 12 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 6 6 6 10 10 10 T T T 13 260-279 10 10 10 16 16 16 F F F 14 280-299 10 10 10 16 16 16 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 10 10 10 14 14 14 T T T 17 340-359 10 10 10 15 15 15 F F F 18 360-379 9 9 9 12 12 12 F F F 19 380-399 10 10 10 18 18 18 F F F 20 400-419 9 9 9 12 12 12 F F F 21 420-439 9 9 9 12 12 12 F F F 22 440-459 6 6 6 10 10 10 F F F 23 460-479 10 10 10 13 13 13 F F F 24 480-499 9 9 9 12 12 12 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 6 6 6 10 10 10 T T T 27 540-559 9 9 9 12 12 12 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 10, Max flush: 18 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 F F F 3 060-079 10 10 10 15 15 15 F F F 4 080-099 11 11 11 19 19 19 F F F 5 100-119 11 11 11 19 19 19 F F F 6 120-139 9 9 9 12 12 12 F F F 7 140-159 9 9 9 12 12 12 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 9 9 9 12 12 12 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 11 11 11 19 19 19 F F F 13 260-279 10 10 10 16 16 16 F F F 14 280-299 10 10 10 16 16 16 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 11 11 11 19 19 19 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 9 9 9 12 12 12 F F F 19 380-399 10 10 10 18 18 18 F F F 20 400-419 9 9 9 12 12 12 F F F 21 420-439 9 9 9 12 12 12 F F F 22 440-459 6 6 6 10 10 10 F F F 23 460-479 10 10 10 13 13 13 F F F 24 480-499 9 9 9 12 12 12 F F F 25 500-519 9 9 9 12 12 12 F F F 26 520-539 11 11 11 19 19 19 F F F 27 540-559 9 9 9 12 12 12 F F F 28 560-579 5 5 5 9 9 9 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 11, Max flush: 19 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.503422847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:19.546925483Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.54746061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.579295301Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.580067915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.580111049Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.580129067Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.580147394Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} {"msg":"Crucible c5a93052-924a-4363-a0d8-91b648e267e4 has session id: fac11916-1b19-4936-b9ae-6bac3cd9d206","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.580837429Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Upstairs opts: Upstairs UUID: c5a93052-924a-4363-a0d8-91b648e267e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.580881633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Crucible stats registered with UUID: c5a93052-924a-4363-a0d8-91b648e267e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.58090672Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:19.580923677Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581290037Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581314404Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} {"msg":"c5a93052-924a-4363-a0d8-91b648e267e4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581377065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581398222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"0","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581426328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"0","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581440476Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"1","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581461373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"1","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.58148629Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"2","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581515256Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"2","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581545462Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581572968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581597465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"ds_connection connected from Ok(127.0.0.1:55781)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581623161Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"ds_connection connected from Ok(127.0.0.1:61893)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.581804626Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"ds_connection connected from Ok(127.0.0.1:65407)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.58184707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.58235577Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"1","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.582545194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"0","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.582601107Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"2","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.583050745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"1","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.583210843Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"2","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.583431482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"0","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.583689177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.58380925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[0]R flush_numbers[0..12]: [8, 7, 7, 15, 19, 19, 12, 12, 5, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584085592Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 10, 11, 11, 9, 9, 3, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.58410654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584121987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[1]R flush_numbers[0..12]: [8, 7, 7, 15, 19, 19, 12, 12, 5, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584135945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 10, 11, 11, 9, 9, 3, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584149874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584164281Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[2]R flush_numbers[0..12]: [8, 7, 7, 15, 19, 19, 12, 12, 5, 12, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.58417825Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 10, 11, 11, 9, 9, 3, 9, 9, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584192158Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584209775Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584223543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584237362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584257209Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584270877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"":"downstairs","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584284505Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584297253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"0","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584311121Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"1","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584325739Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"client":"2","":"downstairs","session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584339468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584352146Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"c5a93052-924a-4363-a0d8-91b648e267e4 is now active with session: fac11916-1b19-4936-b9ae-6bac3cd9d206","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584365354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584377502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240,"session_id":"fac11916-1b19-4936-b9ae-6bac3cd9d206"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584389961Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} {"msg":"Activated sub_volume c5a93052-924a-4363-a0d8-91b648e267e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:19.584401589Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 354 len 32768 data: 3 2 2 2 2 2 2 3 02/30 Write block 213 len 8192 data: 2 2 03/30 Flush 04/30 Write block 484 len 12288 data: 3 3 3 05/30 Write block 255 len 32768 data: 4 4 4 4 4 4 4 4 06/30 Read block 513 len 12288 07/30 Write block 490 len 12288 data: 3 2 2 08/30 Write block 513 len 4096 data: 2 09/30 Read block 571 len 28672 10/30 Write block 396 len 4096 data: 3 11/30 Write block 201 len 8192 data: 2 2 12/30 Read block 239 len 24576 13/30 Read block 215 len 28672 14/30 Read block 504 len 8192 15/30 Read block 271 len 36864 16/30 Read block 29 len 28672 17/30 Read block 261 len 4096 18/30 Read block 361 len 20480 19/30 Read block 22 len 16384 20/30 Read block 579 len 28672 21/30 Write block 352 len 8192 data: 3 3 22/30 Write block 55 len 40960 data: 2 2 2 2 3 3 3 3 2 2 23/30 Write block 565 len 8192 data: 5 5 24/30 Read block 456 len 16384 25/30 Write block 409 len 24576 data: 2 2 2 2 2 2 26/30 Write block 125 len 40960 data: 2 2 2 2 2 2 2 2 2 2 27/30 Write block 337 len 8192 data: 4 4 28/30 Write block 85 len 40960 data: 2 2 2 2 2 2 2 2 3 3 29/30 Read block 341 len 4096 30/30 Read block 350 len 20480 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 3 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 27 26 26 79 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:19.60351452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 7 7 7 T T T 3 060-079 10 10 10 15 15 15 T T T 4 080-099 11 11 11 19 19 19 T T T 5 100-119 11 11 11 19 19 19 F F F 6 120-139 9 9 9 12 12 12 T T T 7 140-159 9 9 9 12 12 12 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 12 12 12 20 20 20 T T T 11 220-239 6 6 6 10 10 10 F F F 12 240-259 11 11 11 19 19 19 T T T 13 260-279 10 10 10 16 16 16 T T T 14 280-299 10 10 10 16 16 16 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 11 11 11 19 19 19 T T T 17 340-359 12 12 12 20 20 20 T T T 18 360-379 12 12 12 20 20 20 F F F 19 380-399 10 10 10 18 18 18 T T T 20 400-419 9 9 9 12 12 12 T T T 21 420-439 9 9 9 12 12 12 F F F 22 440-459 6 6 6 10 10 10 F F F 23 460-479 10 10 10 13 13 13 F F F 24 480-499 9 9 9 12 12 12 T T T 25 500-519 9 9 9 12 12 12 T T T 26 520-539 11 11 11 19 19 19 F F F 27 540-559 9 9 9 12 12 12 F F F 28 560-579 5 5 5 9 9 9 T T T 29 580-599 6 6 6 10 10 10 F F F Max gen: 12, Max flush: 20 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 13 13 13 21 21 21 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 11 11 11 19 19 19 F F F 6 120-139 13 13 13 21 21 21 F F F 7 140-159 9 9 9 12 12 12 F F F 8 160-179 3 3 3 5 5 5 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 13 13 13 21 21 21 F F F 13 260-279 13 13 13 21 21 21 F F F 14 280-299 10 10 10 16 16 16 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 12 12 12 20 20 20 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 13 13 13 21 21 21 F F F 21 420-439 9 9 9 12 12 12 F F F 22 440-459 6 6 6 10 10 10 F F F 23 460-479 10 10 10 13 13 13 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 13 13 13 21 21 21 F F F 26 520-539 11 11 11 19 19 19 F F F 27 540-559 9 9 9 12 12 12 F F F 28 560-579 13 13 13 21 21 21 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 13, Max flush: 21 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.924895631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.970341231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97107263Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.971117264Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97114071Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.971159978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} {"msg":"Crucible 9a651b1a-c10c-49c4-9634-ccc6bdf71d89 has session id: d143eaf1-dc9c-4b55-b0c9-e2073f5892ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.971808499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Upstairs opts: Upstairs UUID: 9a651b1a-c10c-49c4-9634-ccc6bdf71d89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.971853343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Crucible stats registered with UUID: 9a651b1a-c10c-49c4-9634-ccc6bdf71d89","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.971877109Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:23.971898096Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97238289Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972409246Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} {"msg":"9a651b1a-c10c-49c4-9634-ccc6bdf71d89 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972470668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972489575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"0","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972518391Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"0","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972532769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"1","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972547377Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"1","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972561215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"2","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972622127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"2","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972646154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97266876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972693977Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:46145)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972718744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:50536)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97274311Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:56056)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.972957821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.973348127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"0","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.973527452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"1","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.973599712Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"2","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.974194501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"0","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.974224437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"1","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.9744909Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"2","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.974809256Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.974938698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[0]R flush_numbers[0..12]: [8, 7, 21, 21, 21, 19, 21, 12, 5, 12, 21, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975079559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[0]R generation[0..12]: [4, 4, 13, 13, 13, 11, 13, 9, 3, 9, 13, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975101566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975117174Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[1]R flush_numbers[0..12]: [8, 7, 21, 21, 21, 19, 21, 12, 5, 12, 21, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975131262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[1]R generation[0..12]: [4, 4, 13, 13, 13, 11, 13, 9, 3, 9, 13, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97514539Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975160128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[2]R flush_numbers[0..12]: [8, 7, 21, 21, 21, 19, 21, 12, 5, 12, 21, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975174606Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[2]R generation[0..12]: [4, 4, 13, 13, 13, 11, 13, 9, 3, 9, 13, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975188804Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975203292Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97521698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975230558Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975243896Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:01:23.975257284Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"":"downstairs","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975277432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97529211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"0","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975306008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"1","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975319676Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"client":"2","":"downstairs","session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975333594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975346262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"9a651b1a-c10c-49c4-9634-ccc6bdf71d89 is now active with session: d143eaf1-dc9c-4b55-b0c9-e2073f5892ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.97535937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975371709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248,"session_id":"d143eaf1-dc9c-4b55-b0c9-e2073f5892ca"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975384117Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} {"msg":"Activated sub_volume 9a651b1a-c10c-49c4-9634-ccc6bdf71d89","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:23.975395755Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 8192 data: 2 2 02/30 Flush 03/30 Write block 58 len 40960 data: 3 4 4 4 4 3 3 3 3 3 04/30 Read block 391 len 28672 05/30 Read block 123 len 8192 06/30 Write block 156 len 40960 data: 3 3 3 2 2 2 2 2 2 2 07/30 Read block 398 len 36864 08/30 Write block 492 len 16384 data: 3 2 2 2 09/30 Read block 208 len 24576 10/30 Read block 203 len 24576 11/30 Flush 12/30 Flush 13/30 Flush 14/30 Write block 468 len 4096 data: 2 15/30 Flush 16/30 Write block 558 len 40960 data: 3 3 4 4 5 4 5 6 6 5 17/30 Write block 514 len 12288 data: 3 3 2 18/30 Write block 413 len 40960 data: 3 3 2 3 3 3 3 3 3 3 19/30 Write block 512 len 36864 data: 2 3 4 4 3 2 2 2 2 20/30 Read block 449 len 28672 21/30 Write block 494 len 8192 data: 3 3 22/30 Write block 100 len 24576 data: 4 3 2 3 3 3 23/30 Read block 260 len 24576 24/30 Read block 514 len 20480 25/30 Write block 259 len 40960 data: 5 5 5 5 3 2 2 2 2 2 26/30 Write block 505 len 28672 data: 2 2 2 2 2 2 2 27/30 Read block 448 len 8192 28/30 Read block 540 len 8192 29/30 Read block 55 len 40960 30/30 Write block 41 len 12288 data: 3 3 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 10 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:23.995999122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 14 14 14 23 23 23 T T T 3 060-079 14 14 14 23 23 23 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 11 11 11 19 19 19 T T T 6 120-139 13 13 13 21 21 21 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 13 13 13 21 21 21 T T T 13 260-279 13 13 13 21 21 21 T T T 14 280-299 10 10 10 16 16 16 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 12 12 12 20 20 20 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 13 13 13 21 21 21 T T T 21 420-439 9 9 9 12 12 12 T T T 22 440-459 6 6 6 10 10 10 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 14 14 14 23 23 23 T T T 25 500-519 13 13 13 21 21 21 T T T 26 520-539 11 11 11 19 19 19 T T T 27 540-559 9 9 9 12 12 12 T T T 28 560-579 13 13 13 21 21 21 T T T 29 580-599 6 6 6 10 10 10 F F F Max gen: 14, Max flush: 26 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 15 15 15 27 27 27 F F F 3 060-079 14 14 14 23 23 23 F F F 4 080-099 13 13 13 21 21 21 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 13 13 13 21 21 21 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 6 6 6 10 10 10 F F F 12 240-259 15 15 15 27 27 27 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 10 10 10 16 16 16 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 13 13 13 21 21 21 F F F 18 360-379 12 12 12 20 20 20 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 15 15 15 27 27 27 F F F 21 420-439 15 15 15 27 27 27 F F F 22 440-459 6 6 6 10 10 10 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 15 15 15 27 27 27 F F F 25 500-519 15 15 15 27 27 27 F F F 26 520-539 15 15 15 27 27 27 F F F 27 540-559 15 15 15 27 27 27 F F F 28 560-579 15 15 15 27 27 27 F F F 29 580-599 6 6 6 10 10 10 F F F Max gen: 15, Max flush: 27 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.276137431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:38.320992452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.321577641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.354356973Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.355219354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.355262088Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.355279636Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.355297923Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} {"msg":"Crucible af5d8c9b-72df-4a16-9713-fe7119fbe3b4 has session id: fb2830ee-cfb2-4a97-b15b-3590b455d488","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.355849647Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Upstairs opts: Upstairs UUID: af5d8c9b-72df-4a16-9713-fe7119fbe3b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.355966931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Crucible stats registered with UUID: af5d8c9b-72df-4a16-9713-fe7119fbe3b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.355992558Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:38.356011265Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356505117Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356538762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} {"msg":"af5d8c9b-72df-4a16-9713-fe7119fbe3b4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356595015Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356622491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"0","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356654037Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"0","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356669564Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"1","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356684802Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"1","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.35669898Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"2","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356713989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"2","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356727416Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356746994Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356770411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"ds_connection connected from Ok(127.0.0.1:35635)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356861718Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"ds_connection connected from Ok(127.0.0.1:47844)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356904122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"ds_connection connected from Ok(127.0.0.1:44333)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.356934648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.35786308Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"1","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.357893526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"0","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.357909674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"2","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.358636614Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"0","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.358677278Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"1","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.358710244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"2","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359299863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359335548Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[0]R flush_numbers[0..12]: [8, 7, 27, 23, 21, 27, 21, 23, 23, 12, 21, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.35939065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[0]R generation[0..12]: [4, 4, 15, 14, 13, 15, 13, 14, 14, 9, 13, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359420006Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359437254Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[1]R flush_numbers[0..12]: [8, 7, 27, 23, 21, 27, 21, 23, 23, 12, 21, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359451832Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[1]R generation[0..12]: [4, 4, 15, 14, 13, 15, 13, 14, 14, 9, 13, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.35946596Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359480408Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[2]R flush_numbers[0..12]: [8, 7, 27, 23, 21, 27, 21, 23, 23, 12, 21, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359498366Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[2]R generation[0..12]: [4, 4, 15, 14, 13, 15, 13, 14, 14, 9, 13, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359522292Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359549888Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359572705Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:38.359587553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359606551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359620069Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"":"downstairs","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359633657Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359646755Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"0","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359660743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"1","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359674301Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"client":"2","":"downstairs","session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359687919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359700458Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"af5d8c9b-72df-4a16-9713-fe7119fbe3b4 is now active with session: fb2830ee-cfb2-4a97-b15b-3590b455d488","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359713526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359725954Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256,"session_id":"fb2830ee-cfb2-4a97-b15b-3590b455d488"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359738313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} {"msg":"Activated sub_volume af5d8c9b-72df-4a16-9713-fe7119fbe3b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:38.359752631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 24576 data: 3 4 4 3 2 2 02/30 Write block 286 len 32768 data: 3 3 2 2 2 3 3 2 03/30 Write block 353 len 4096 data: 4 04/30 Write block 58 len 32768 data: 4 5 5 5 5 4 4 4 05/30 Read block 598 len 4096 06/30 Read block 514 len 24576 07/30 Write block 93 len 28672 data: 4 4 3 3 3 3 4 08/30 Write block 135 len 4096 data: 2 09/30 Write block 552 len 12288 data: 5 5 5 10/30 Write block 55 len 4096 data: 3 11/30 Write block 440 len 24576 data: 2 3 3 3 2 2 12/30 Write block 416 len 40960 data: 4 4 4 4 4 4 4 3 3 3 13/30 Write block 578 len 40960 data: 2 2 2 2 2 2 3 3 3 3 14/30 Read block 262 len 36864 15/30 Read block 99 len 40960 16/30 Write block 68 len 24576 data: 3 4 3 2 2 2 17/30 Read block 395 len 40960 18/30 Write block 343 len 20480 data: 2 2 2 2 2 19/30 Read block 538 len 24576 20/30 Write block 238 len 36864 data: 2 2 2 2 2 3 3 3 3 21/30 Write block 22 len 24576 data: 2 2 2 2 2 2 22/30 Write block 356 len 24576 data: 3 3 3 3 3 4 23/30 Write block 333 len 40960 data: 3 4 4 4 5 5 3 3 3 3 24/30 Write block 297 len 36864 data: 2 2 2 2 2 2 2 2 2 25/30 Read block 177 len 8192 26/30 Read block 169 len 16384 27/30 Write block 254 len 8192 data: 4 5 28/30 Read block 29 len 36864 29/30 Read block 539 len 28672 30/30 Read block 187 len 20480 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 30 29 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:38.378921874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 4 4 4 7 7 7 T T T 2 040-059 15 15 15 27 27 27 T T T 3 060-079 14 14 14 23 23 23 T T T 4 080-099 13 13 13 21 21 21 T T T 5 100-119 15 15 15 27 27 27 F F F 6 120-139 13 13 13 21 21 21 T T T 7 140-159 14 14 14 23 23 23 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 6 6 6 10 10 10 T T T 12 240-259 15 15 15 27 27 27 T T T 13 260-279 15 15 15 27 27 27 F F F 14 280-299 10 10 10 16 16 16 T T T 15 300-319 1 1 1 1 1 1 T T T 16 320-339 13 13 13 21 21 21 T T T 17 340-359 13 13 13 21 21 21 T T T 18 360-379 12 12 12 20 20 20 T T T 19 380-399 13 13 13 21 21 21 F F F 20 400-419 15 15 15 27 27 27 T T T 21 420-439 15 15 15 27 27 27 T T T 22 440-459 6 6 6 10 10 10 T T T 23 460-479 14 14 14 26 26 26 F F F 24 480-499 15 15 15 27 27 27 F F F 25 500-519 15 15 15 27 27 27 F F F 26 520-539 15 15 15 27 27 27 F F F 27 540-559 15 15 15 27 27 27 T T T 28 560-579 15 15 15 27 27 27 T T T 29 580-599 6 6 6 10 10 10 T T T Max gen: 15, Max flush: 27 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 8 8 8 F F F 1 020-039 16 16 16 28 28 28 F F F 2 040-059 16 16 16 28 28 28 F F F 3 060-079 16 16 16 28 28 28 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 16 16 16 28 28 28 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 14 14 14 23 23 23 F F F 9 180-199 9 9 9 12 12 12 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 16 16 16 28 28 28 F F F 12 240-259 16 16 16 28 28 28 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 16 16 16 28 28 28 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 28 28 28 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 15 15 15 27 27 27 F F F 25 500-519 15 15 15 27 27 27 F F F 26 520-539 15 15 15 27 27 27 F F F 27 540-559 16 16 16 28 28 28 F F F 28 560-579 16 16 16 28 28 28 F F F 29 580-599 16 16 16 28 28 28 F F F Max gen: 16, Max flush: 28 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.719808152Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.764136645Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.764882083Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.764917048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.764936815Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.764957932Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} {"msg":"Crucible acfaed82-3682-418b-998c-3dddf41d872e has session id: c7ddf800-5c0b-43b3-a35a-388b9f91f877","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.765578427Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Upstairs opts: Upstairs UUID: acfaed82-3682-418b-998c-3dddf41d872e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.76562441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Crucible stats registered with UUID: acfaed82-3682-418b-998c-3dddf41d872e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.765647987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:42.765669154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766177065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766202181Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} {"msg":"acfaed82-3682-418b-998c-3dddf41d872e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766278491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766296768Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"0","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766326584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"0","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766345381Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"1","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766360549Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"1","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766374127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"2","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766388515Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"2","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766402154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766416741Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766431139Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"ds_connection connected from Ok(127.0.0.1:64371)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766610745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"ds_connection connected from Ok(127.0.0.1:34029)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766656408Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"ds_connection connected from Ok(127.0.0.1:47356)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.766692014Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.767256066Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"1","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.767285472Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"0","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.76795375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"1","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.768120437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"0","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.768605411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.768717755Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.768749041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"2","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769286257Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"2","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[0]R flush_numbers[0..12]: [8, 28, 28, 28, 28, 27, 28, 23, 23, 12, 21, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.76977085Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[0]R generation[0..12]: [4, 16, 16, 16, 16, 15, 16, 14, 14, 9, 13, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769793837Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769809775Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[1]R flush_numbers[0..12]: [8, 28, 28, 28, 28, 27, 28, 23, 23, 12, 21, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769824323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[1]R generation[0..12]: [4, 16, 16, 16, 16, 15, 16, 14, 14, 9, 13, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769838781Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769853409Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[2]R flush_numbers[0..12]: [8, 28, 28, 28, 28, 27, 28, 23, 23, 12, 21, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769867657Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[2]R generation[0..12]: [4, 16, 16, 16, 16, 15, 16, 14, 14, 9, 13, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769881875Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769896363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Max found gen is 17","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769909821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Generation requested: 18 >= found:17","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769928829Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769950455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769964124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"":"downstairs","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.769977882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.76999021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"0","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770003938Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"1","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770017446Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"client":"2","":"downstairs","session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770031214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770043813Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"acfaed82-3682-418b-998c-3dddf41d872e is now active with session: c7ddf800-5c0b-43b3-a35a-388b9f91f877","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770057021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770069319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264,"session_id":"c7ddf800-5c0b-43b3-a35a-388b9f91f877"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770081687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} {"msg":"Activated sub_volume acfaed82-3682-418b-998c-3dddf41d872e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:42.770093506Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 498 len 28672 data: 3 3 2 2 2 2 2 02/30 Read block 244 len 12288 03/30 Read block 159 len 28672 04/30 Flush 05/30 Write block 6 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Write block 235 len 28672 data: 2 2 3 3 3 3 3 07/30 Write block 578 len 36864 data: 3 3 3 3 3 3 4 4 4 08/30 Read block 139 len 32768 09/30 Write block 39 len 24576 data: 3 4 5 5 4 3 10/30 Write block 60 len 32768 data: 6 6 6 5 5 5 4 4 11/30 Flush 12/30 Write block 42 len 24576 data: 6 5 4 3 2 3 13/30 Flush 14/30 Write block 121 len 12288 data: 4 4 2 15/30 Write block 230 len 36864 data: 3 3 3 3 2 3 3 4 4 16/30 Write block 377 len 8192 data: 3 2 17/30 Read block 381 len 24576 18/30 Write block 480 len 24576 data: 2 2 3 3 4 4 19/30 Read block 402 len 32768 20/30 Read block 40 len 40960 21/30 Read block 149 len 36864 22/30 Read block 525 len 28672 23/30 Write block 164 len 4096 data: 3 24/30 Read block 390 len 12288 25/30 Write block 311 len 24576 data: 2 2 2 2 2 2 26/30 Read block 44 len 4096 27/30 Write block 194 len 8192 data: 3 2 28/30 Read block 49 len 40960 29/30 Read block 232 len 28672 30/30 Read block 460 len 12288 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 3 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 17 16 17 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:42.791770464Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 30 30 30 F F F 1 020-039 18 18 18 30 30 30 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 18 18 18 30 30 30 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 16 16 16 28 28 28 T T T 7 140-159 14 14 14 23 23 23 F F F 8 160-179 14 14 14 23 23 23 T T T 9 180-199 9 9 9 12 12 12 T T T 10 200-219 13 13 13 21 21 21 F F F 11 220-239 18 18 18 30 30 30 T T T 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 16 16 16 28 28 28 F F F 15 300-319 16 16 16 28 28 28 T T T 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 16 16 16 28 28 28 T T T 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 28 28 28 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 18 18 18 29 29 29 T T T 25 500-519 18 18 18 29 29 29 F F F 26 520-539 15 15 15 27 27 27 F F F 27 540-559 16 16 16 28 28 28 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 18 18 18 30 30 30 F F F Max gen: 18, Max flush: 31 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 30 30 30 F F F 1 020-039 18 18 18 30 30 30 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 18 18 18 30 30 30 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 19 19 19 32 32 32 F F F 9 180-199 19 19 19 32 32 32 F F F 10 200-219 13 13 13 21 21 21 F F F 11 220-239 19 19 19 32 32 32 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 16 16 16 28 28 28 F F F 15 300-319 19 19 19 32 32 32 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 28 28 28 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 19 19 19 32 32 32 F F F 25 500-519 18 18 18 29 29 29 F F F 26 520-539 15 15 15 27 27 27 F F F 27 540-559 16 16 16 28 28 28 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 18 18 18 30 30 30 F F F Max gen: 19, Max flush: 32 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.049259693Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:47.092345217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.092738283Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.124613969Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.125290166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.12532777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.125343159Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.125358256Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} {"msg":"Crucible 457b2bae-df40-49e6-a005-7d753be8793f has session id: 8c09c7be-cb16-4f51-8703-d8b6f84a681b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.125962103Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Upstairs opts: Upstairs UUID: 457b2bae-df40-49e6-a005-7d753be8793f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126011847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Crucible stats registered with UUID: 457b2bae-df40-49e6-a005-7d753be8793f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126029784Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:47.126043402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126465464Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126490881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} {"msg":"457b2bae-df40-49e6-a005-7d753be8793f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126553072Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.12657057Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"0","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126600615Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"0","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126616013Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"1","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126631141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"1","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126645039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"2","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126665686Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"2","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126687283Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.12671144Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126741486Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58636)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126769192Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64739)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.12685885Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46192)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.126906593Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.127706833Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"1","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.127756387Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"2","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.127865881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"0","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.128401688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"2","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.128508003Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"1","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.12852894Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"0","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129012324Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129129958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[0]R flush_numbers[0..12]: [30, 30, 31, 30, 28, 27, 32, 23, 32, 32, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129197108Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[0]R generation[0..12]: [18, 18, 18, 18, 16, 15, 19, 14, 19, 19, 13, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129216876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129235713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[1]R flush_numbers[0..12]: [30, 30, 31, 30, 28, 27, 32, 23, 32, 32, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129250551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[1]R generation[0..12]: [18, 18, 18, 18, 16, 15, 19, 14, 19, 19, 13, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129265009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129279517Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[2]R flush_numbers[0..12]: [30, 30, 31, 30, 28, 27, 32, 23, 32, 32, 21, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129293785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[2]R generation[0..12]: [18, 18, 18, 18, 16, 15, 19, 14, 19, 19, 13, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129308403Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129323011Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129336549Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129350097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129363646Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129377193Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"":"downstairs","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129397651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129409999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"0","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129424117Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"1","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129438075Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"client":"2","":"downstairs","session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129454213Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129467151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"457b2bae-df40-49e6-a005-7d753be8793f is now active with session: 8c09c7be-cb16-4f51-8703-d8b6f84a681b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129480559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129492988Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272,"session_id":"8c09c7be-cb16-4f51-8703-d8b6f84a681b"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129505546Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} {"msg":"Activated sub_volume 457b2bae-df40-49e6-a005-7d753be8793f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:47.129517124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 20480 data: 2 2 2 2 2 02/30 Write block 280 len 20480 data: 3 3 3 3 3 03/30 Read block 551 len 36864 04/30 Write block 86 len 20480 data: 3 3 3 3 3 05/30 Write block 91 len 36864 data: 3 3 5 5 4 4 4 4 5 06/30 Read block 248 len 16384 07/30 Read block 48 len 16384 08/30 Read block 107 len 32768 09/30 Read block 10 len 28672 10/30 Read block 131 len 8192 11/30 Write block 292 len 36864 data: 4 3 2 2 2 3 3 3 3 12/30 Flush 13/30 Write block 204 len 20480 data: 3 3 3 3 3 14/30 Write block 585 len 24576 data: 5 5 4 4 3 3 15/30 Write block 528 len 36864 data: 3 3 2 2 2 2 2 2 2 16/30 Read block 427 len 16384 17/30 Write block 198 len 12288 data: 2 2 2 18/30 Read block 303 len 40960 19/30 Write block 512 len 36864 data: 3 4 5 5 4 3 3 3 3 20/30 Read block 580 len 36864 21/30 Read block 39 len 36864 22/30 Read block 455 len 24576 23/30 Read block 33 len 40960 24/30 Read block 392 len 20480 25/30 Read block 431 len 40960 26/30 Read block 173 len 12288 27/30 Read block 213 len 40960 28/30 Write block 62 len 20480 data: 7 6 6 6 5 29/30 Read block 246 len 28672 30/30 Read block 13 len 32768 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:47.152107487Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 33 33 33 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 18 18 18 30 30 30 T T T 4 080-099 20 20 20 33 33 33 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 19 19 19 32 32 32 F F F 9 180-199 19 19 19 32 32 32 T T T 10 200-219 13 13 13 21 21 21 T T T 11 220-239 19 19 19 32 32 32 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 20 20 20 33 33 33 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 28 28 28 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 19 19 19 32 32 32 F F F 25 500-519 18 18 18 29 29 29 T T T 26 520-539 15 15 15 27 27 27 T T T 27 540-559 16 16 16 28 28 28 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 18 18 18 30 30 30 T T T Max gen: 20, Max flush: 33 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 33 33 33 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 21 21 21 34 34 34 F F F 4 080-099 20 20 20 33 33 33 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 19 19 19 32 32 32 F F F 9 180-199 21 21 21 34 34 34 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 19 19 19 32 32 32 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 20 20 20 33 33 33 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 28 28 28 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 19 19 19 32 32 32 F F F 25 500-519 21 21 21 34 34 34 F F F 26 520-539 21 21 21 34 34 34 F F F 27 540-559 16 16 16 28 28 28 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 21 21 21 34 34 34 F F F Max gen: 21, Max flush: 34 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.431546237Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.476210734Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.476956081Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.476992186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.477015473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.47703375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} {"msg":"Crucible e2bd6f7f-d8a7-4368-ada2-212df8492660 has session id: d76e4707-b75e-4952-ac89-e791985a85fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.477604032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Upstairs opts: Upstairs UUID: e2bd6f7f-d8a7-4368-ada2-212df8492660, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.477648526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Crucible stats registered with UUID: e2bd6f7f-d8a7-4368-ada2-212df8492660","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.477672043Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:51.477686251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478163365Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478189982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} {"msg":"e2bd6f7f-d8a7-4368-ada2-212df8492660 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478256872Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.47827675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"0","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478310825Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"0","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478326123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"1","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478341051Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"1","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478354589Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"2","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478377566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"2","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478403312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478426989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478466873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65236)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478484471Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37734)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478509078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:46142)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.478535694Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.479215021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"1","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.479339653Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"0","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.479750507Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"1","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.479860582Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"0","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.480348675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.480421315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.480927515Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"2","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.481485698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"2","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[0]R flush_numbers[0..12]: [33, 33, 31, 34, 33, 27, 32, 23, 32, 34, 34, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482025534Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[0]R generation[0..12]: [20, 20, 18, 21, 20, 15, 19, 14, 19, 21, 21, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.48205104Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482066908Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[1]R flush_numbers[0..12]: [33, 33, 31, 34, 33, 27, 32, 23, 32, 34, 34, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482081746Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[1]R generation[0..12]: [20, 20, 18, 21, 20, 15, 19, 14, 19, 21, 21, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482095614Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482110242Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[2]R flush_numbers[0..12]: [33, 33, 31, 34, 33, 27, 32, 23, 32, 34, 34, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.48212516Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[2]R generation[0..12]: [20, 20, 18, 21, 20, 15, 19, 14, 19, 21, 21, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482139498Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482153936Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482171184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Generation requested: 22 >= found:22","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482191751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482205489Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482218688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"":"downstairs","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482232275Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482244544Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"0","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482258262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"1","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.48227193Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"client":"2","":"downstairs","session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482285548Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482298237Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"e2bd6f7f-d8a7-4368-ada2-212df8492660 is now active with session: d76e4707-b75e-4952-ac89-e791985a85fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482311525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482323973Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280,"session_id":"d76e4707-b75e-4952-ac89-e791985a85fd"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.482336551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} {"msg":"Activated sub_volume e2bd6f7f-d8a7-4368-ada2-212df8492660","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:51.4823483Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 280 len 20480 data: 4 4 4 4 4 02/30 Write block 154 len 16384 data: 3 3 4 4 03/30 Write block 468 len 4096 data: 3 04/30 Write block 75 len 28672 data: 2 2 2 2 2 2 3 05/30 Flush 06/30 Write block 76 len 36864 data: 3 3 3 3 3 4 2 2 2 07/30 Write block 529 len 36864 data: 4 3 3 3 3 3 3 3 2 08/30 Read block 504 len 12288 09/30 Read block 241 len 32768 10/30 Flush 11/30 Write block 483 len 28672 data: 4 5 5 4 3 3 3 12/30 Write block 220 len 36864 data: 3 4 4 4 4 4 5 4 3 13/30 Read block 111 len 24576 14/30 Read block 34 len 16384 15/30 Read block 505 len 24576 16/30 Read block 189 len 36864 17/30 Write block 228 len 12288 data: 4 3 4 18/30 Read block 545 len 12288 19/30 Read block 381 len 24576 20/30 Write block 221 len 16384 data: 5 5 5 5 21/30 Read block 365 len 36864 22/30 Read block 511 len 32768 23/30 Read block 88 len 32768 24/30 Write block 405 len 16384 data: 2 2 2 2 25/30 Read block 146 len 32768 26/30 Write block 290 len 8192 data: 3 4 27/30 Write block 162 len 40960 data: 3 3 4 3 2 2 2 3 3 3 28/30 Write block 521 len 24576 data: 3 3 3 3 4 3 29/30 Read block 503 len 20480 30/30 Read block 485 len 24576 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 20 19 19 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:51.502854649Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 33 33 33 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 22 22 22 36 36 36 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 22 22 22 35 35 35 F F F 8 160-179 19 19 19 32 32 32 T T T 9 180-199 21 21 21 34 34 34 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 19 19 19 32 32 32 T T T 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 22 22 22 35 35 35 T T T 15 300-319 20 20 20 33 33 33 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 16 16 16 28 28 28 T T T 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 22 22 22 35 35 35 F F F 24 480-499 19 19 19 32 32 32 T T T 25 500-519 21 21 21 34 34 34 F F F 26 520-539 22 22 22 36 36 36 T T T 27 540-559 16 16 16 28 28 28 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 21 21 21 34 34 34 F F F Max gen: 22, Max flush: 36 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 33 33 33 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 22 22 22 36 36 36 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 22 22 22 35 35 35 F F F 8 160-179 23 23 23 37 37 37 F F F 9 180-199 21 21 21 34 34 34 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 23 23 23 37 37 37 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 20 20 20 33 33 33 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 23 23 23 37 37 37 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 16 16 16 28 28 28 F F F 23 460-479 22 22 22 35 35 35 F F F 24 480-499 23 23 23 37 37 37 F F F 25 500-519 21 21 21 34 34 34 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 16 16 16 28 28 28 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 21 21 21 34 34 34 F F F Max gen: 23, Max flush: 37 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.752792237Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.797761791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.798502919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.798539034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.798553722Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.79856857Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} {"msg":"Crucible 528545de-1dcb-45c2-9f69-32741c5ba02d has session id: 309603b7-f706-4d4f-81d0-f55422429445","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799082729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Upstairs opts: Upstairs UUID: 528545de-1dcb-45c2-9f69-32741c5ba02d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799128093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Crucible stats registered with UUID: 528545de-1dcb-45c2-9f69-32741c5ba02d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.79915176Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:55.799189395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799762236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799794542Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} {"msg":"528545de-1dcb-45c2-9f69-32741c5ba02d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799857433Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799885439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"0","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799916625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"0","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799931723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"1","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799946651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"1","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799961009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"2","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799976157Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"2","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.799989745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.800008042Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.80002311Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"ds_connection connected from Ok(127.0.0.1:36544)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.800186168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"ds_connection connected from Ok(127.0.0.1:42901)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.800228602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"ds_connection connected from Ok(127.0.0.1:51328)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.800356434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.800939124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"0","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.800975949Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"1","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.801646247Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"1","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.801784668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"0","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.801937077Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"2","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.802162926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.802412562Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.802491151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"2","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[0]R flush_numbers[0..12]: [33, 33, 31, 36, 36, 27, 32, 35, 37, 34, 34, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.802988552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[0]R generation[0..12]: [20, 20, 18, 22, 22, 15, 19, 22, 23, 21, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803012659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803031217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[1]R flush_numbers[0..12]: [33, 33, 31, 36, 36, 27, 32, 35, 37, 34, 34, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803046184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[1]R generation[0..12]: [20, 20, 18, 22, 22, 15, 19, 22, 23, 21, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803060402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803074651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[2]R flush_numbers[0..12]: [33, 33, 31, 36, 36, 27, 32, 35, 37, 34, 34, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803088689Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[2]R generation[0..12]: [20, 20, 18, 22, 22, 15, 19, 22, 23, 21, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803102777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803117055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803130593Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803144291Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803160229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803177236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"":"downstairs","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803194244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803209752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"0","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.80322689Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"1","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803244057Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"client":"2","":"downstairs","session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:55.803263454Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803283072Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"528545de-1dcb-45c2-9f69-32741c5ba02d is now active with session: 309603b7-f706-4d4f-81d0-f55422429445","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803299509Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803314727Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288,"session_id":"309603b7-f706-4d4f-81d0-f55422429445"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803330305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} {"msg":"Activated sub_volume 528545de-1dcb-45c2-9f69-32741c5ba02d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:55.803345003Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 6 len 4096 data: 3 02/30 Read block 472 len 24576 03/30 Write block 479 len 28672 data: 3 3 3 4 5 6 6 04/30 Flush 05/30 Write block 168 len 28672 data: 3 4 4 4 3 3 2 06/30 Read block 348 len 40960 07/30 Write block 400 len 24576 data: 3 3 3 3 3 3 08/30 Read block 294 len 20480 09/30 Write block 304 len 4096 data: 3 10/30 Write block 446 len 20480 data: 2 2 2 2 2 11/30 Read block 543 len 40960 12/30 Read block 133 len 8192 13/30 Read block 239 len 20480 14/30 Read block 136 len 28672 15/30 Read block 125 len 36864 16/30 Read block 468 len 8192 17/30 Read block 295 len 28672 18/30 Read block 268 len 20480 19/30 Write block 162 len 16384 data: 4 4 5 4 20/30 Flush 21/30 Write block 543 len 36864 data: 2 2 2 2 3 3 4 4 4 22/30 Read block 206 len 40960 23/30 Read block 106 len 4096 24/30 Read block 568 len 24576 25/30 Flush 26/30 Write block 235 len 12288 data: 4 4 5 27/30 Write block 445 len 28672 data: 3 3 3 3 3 3 2 28/30 Write block 144 len 12288 data: 2 2 2 29/30 Read block 382 len 20480 30/30 Read block 15 len 4096 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:55.824514282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 38 38 38 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 22 22 22 36 36 36 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 22 22 22 35 35 35 T T T 8 160-179 24 24 24 39 39 39 F F F 9 180-199 21 21 21 34 34 34 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 23 23 23 37 37 37 T T T 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 24 24 24 39 39 39 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 24 24 24 39 39 39 T T T 23 460-479 24 24 24 38 38 38 F F F 24 480-499 24 24 24 38 38 38 F F F 25 500-519 21 21 21 34 34 34 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 24 24 24 40 40 40 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 21 21 21 34 34 34 F F F Max gen: 24, Max flush: 40 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 38 38 38 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 22 22 22 36 36 36 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 25 25 25 41 41 41 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 21 21 21 34 34 34 F F F 10 200-219 21 21 21 34 34 34 F F F 11 220-239 25 25 25 41 41 41 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 24 24 24 39 39 39 F F F 16 320-339 16 16 16 28 28 28 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 25 25 25 41 41 41 F F F 23 460-479 24 24 24 38 38 38 F F F 24 480-499 24 24 24 38 38 38 F F F 25 500-519 21 21 21 34 34 34 F F F 26 520-539 23 23 23 37 37 37 F F F 27 540-559 24 24 24 40 40 40 F F F 28 560-579 18 18 18 30 30 30 F F F 29 580-599 21 21 21 34 34 34 F F F Max gen: 25, Max flush: 41 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.060838881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.106631363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.107315259Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.107352134Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.107366801Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.10738193Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} {"msg":"Crucible 411afac3-0288-4a74-9444-b8afb308915b has session id: 7c956509-5bbd-45f4-8fd1-62389cc56ae8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.10796012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Upstairs opts: Upstairs UUID: 411afac3-0288-4a74-9444-b8afb308915b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108004234Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Crucible stats registered with UUID: 411afac3-0288-4a74-9444-b8afb308915b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.10803179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:00.108054177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108504565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108537251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} {"msg":"411afac3-0288-4a74-9444-b8afb308915b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108588393Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.10861553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"0","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108655744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"0","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.10868602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"1","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108717326Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"1","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108733904Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.10875694Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108782547Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108828251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108858676Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:00.108891892Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:00.108924027Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"ds_connection connected from Ok(127.0.0.1:64340)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.108952123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:00.108983519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"ds_connection connected from Ok(127.0.0.1:35629)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.109005046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.109069687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.109531454Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"0","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.109568909Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"1","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.110092757Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"1","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.110209071Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"0","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.110596767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:00.110801469Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.110712396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63484)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.110809632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.111920939Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.112460015Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[0]R flush_numbers[0..12]: [38, 33, 31, 36, 36, 27, 32, 41, 39, 34, 34, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113040485Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[0]R generation[0..12]: [24, 20, 18, 22, 22, 15, 19, 25, 24, 21, 21, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113062622Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.11307874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[1]R flush_numbers[0..12]: [38, 33, 31, 36, 36, 27, 32, 41, 39, 34, 34, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113093618Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[1]R generation[0..12]: [24, 20, 18, 22, 22, 15, 19, 25, 24, 21, 21, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113108176Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113122594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[2]R flush_numbers[0..12]: [38, 33, 31, 36, 36, 27, 32, 41, 39, 34, 34, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113141941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[2]R generation[0..12]: [24, 20, 18, 22, 22, 15, 19, 25, 24, 21, 21, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113156479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113171097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113184776Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113198274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113211832Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.11322505Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"":"downstairs","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113238618Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:02:10.113251126Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"0","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113270974Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"1","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113284852Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"client":"2","":"downstairs","session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.11329868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113311358Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"411afac3-0288-4a74-9444-b8afb308915b is now active with session: 7c956509-5bbd-45f4-8fd1-62389cc56ae8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113324526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113336715Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296,"session_id":"7c956509-5bbd-45f4-8fd1-62389cc56ae8"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113349233Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} {"msg":"Activated sub_volume 411afac3-0288-4a74-9444-b8afb308915b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:10.113362721Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 103 len 8192 data: 4 4 02/30 Write block 530 len 40960 data: 4 4 4 4 4 4 4 3 2 2 03/30 Write block 318 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 365 len 40960 05/30 Read block 55 len 24576 06/30 Write block 553 len 40960 data: 6 6 5 3 3 4 4 5 5 6 07/30 Read block 579 len 16384 08/30 Write block 86 len 36864 data: 4 4 4 4 4 4 4 6 6 09/30 Write block 520 len 12288 data: 4 4 4 10/30 Flush 11/30 Write block 339 len 12288 data: 4 4 4 12/30 Write block 178 len 8192 data: 2 2 13/30 Read block 528 len 16384 14/30 Write block 188 len 24576 data: 2 3 3 3 3 3 15/30 Read block 366 len 20480 16/30 Read block 336 len 8192 17/30 Read block 232 len 16384 18/30 Write block 209 len 40960 data: 3 3 2 2 3 3 2 2 3 3 19/30 Read block 452 len 4096 20/30 Read block 538 len 36864 21/30 Flush 22/30 Write block 490 len 12288 data: 4 3 4 23/30 Read block 289 len 8192 24/30 Write block 237 len 16384 data: 6 5 4 4 25/30 Write block 219 len 20480 data: 3 4 6 6 6 26/30 Read block 190 len 20480 27/30 Read block 187 len 28672 28/30 Write block 248 len 4096 data: 2 29/30 Read block 486 len 40960 30/30 Read block 217 len 36864 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 8 9 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:10.135141416Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 38 38 38 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 26 26 26 42 42 42 F F F 5 100-119 26 26 26 42 42 42 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 25 25 25 41 41 41 F F F 8 160-179 26 26 26 43 43 43 F F F 9 180-199 26 26 26 43 43 43 F F F 10 200-219 26 26 26 43 43 43 T T T 11 220-239 25 25 25 41 41 41 T T T 12 240-259 18 18 18 30 30 30 T T T 13 260-279 15 15 15 27 27 27 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 26 26 26 43 43 43 F F F 17 340-359 26 26 26 43 43 43 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 25 25 25 41 41 41 F F F 23 460-479 24 24 24 38 38 38 F F F 24 480-499 24 24 24 38 38 38 T T T 25 500-519 21 21 21 34 34 34 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 26 26 26 42 42 42 F F F 28 560-579 26 26 26 42 42 42 F F F 29 580-599 21 21 21 34 34 34 F F F Max gen: 26, Max flush: 43 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 38 38 38 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 18 18 18 31 31 31 F F F 3 060-079 22 22 22 36 36 36 F F F 4 080-099 26 26 26 42 42 42 F F F 5 100-119 26 26 26 42 42 42 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 25 25 25 41 41 41 F F F 8 160-179 26 26 26 43 43 43 F F F 9 180-199 26 26 26 43 43 43 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 27 27 27 44 44 44 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 23 23 23 37 37 37 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 26 26 26 43 43 43 F F F 17 340-359 26 26 26 43 43 43 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 25 25 25 41 41 41 F F F 23 460-479 24 24 24 38 38 38 F F F 24 480-499 27 27 27 44 44 44 F F F 25 500-519 21 21 21 34 34 34 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 26 26 26 42 42 42 F F F 28 560-579 26 26 26 42 42 42 F F F 29 580-599 21 21 21 34 34 34 F F F Max gen: 27, Max flush: 44 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.379294388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.424753535Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.425640593Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.425687287Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.425705264Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.425723512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} {"msg":"Crucible 45444af7-1579-4710-bef6-b864d3a45756 has session id: c95c2dc0-19a5-48c8-86c5-af0ab5dca06e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.426341697Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Upstairs opts: Upstairs UUID: 45444af7-1579-4710-bef6-b864d3a45756, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.426400049Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Crucible stats registered with UUID: 45444af7-1579-4710-bef6-b864d3a45756","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.426426675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:14.42646625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.42704459Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427072486Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} {"msg":"45444af7-1579-4710-bef6-b864d3a45756 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427145016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427163434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"0","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.4271933Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"0","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427208078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"1","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427223165Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"1","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427237424Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"2","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427254501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"2","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427339849Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427376435Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427405191Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34463)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427435127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43824)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427464052Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36941)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.427587355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.428071449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"2","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.428101245Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"0","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.428723439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"0","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.428868319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"2","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.429199104Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"1","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.429412914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.42951529Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.429770835Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"1","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[0]R flush_numbers[0..12]: [38, 33, 31, 36, 42, 42, 32, 41, 43, 43, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430288844Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[0]R generation[0..12]: [24, 20, 18, 22, 26, 26, 19, 25, 26, 26, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.4303196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430339787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[1]R flush_numbers[0..12]: [38, 33, 31, 36, 42, 42, 32, 41, 43, 43, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430357834Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[1]R generation[0..12]: [24, 20, 18, 22, 26, 26, 19, 25, 26, 26, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430375492Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430395229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[2]R flush_numbers[0..12]: [38, 33, 31, 36, 42, 42, 32, 41, 43, 43, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430413137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[2]R generation[0..12]: [24, 20, 18, 22, 26, 26, 19, 25, 26, 26, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430430764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430448742Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.43046571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430482987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430499665Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430516563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"":"downstairs","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.43053341Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:02:14.430548738Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"0","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430571125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"1","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430590642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"client":"2","":"downstairs","session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.43060806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430623958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"45444af7-1579-4710-bef6-b864d3a45756 is now active with session: c95c2dc0-19a5-48c8-86c5-af0ab5dca06e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430640346Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430655853Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304,"session_id":"c95c2dc0-19a5-48c8-86c5-af0ab5dca06e"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430671451Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} {"msg":"Activated sub_volume 45444af7-1579-4710-bef6-b864d3a45756","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:14.430685889Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 563 len 4096 data: 5 02/30 Read block 304 len 36864 03/30 Write block 403 len 4096 data: 4 04/30 Read block 116 len 32768 05/30 Write block 4 len 8192 data: 2 2 06/30 Write block 555 len 28672 data: 6 4 4 5 5 6 6 07/30 Read block 466 len 32768 08/30 Write block 78 len 32768 data: 4 4 4 5 3 3 3 3 09/30 Write block 259 len 24576 data: 6 6 6 6 4 3 10/30 Write block 51 len 20480 data: 2 2 2 2 4 11/30 Read block 238 len 20480 12/30 Read block 294 len 4096 13/30 Read block 321 len 16384 14/30 Flush 15/30 Write block 590 len 40960 data: 4 2 2 2 3 2 2 2 2 2 16/30 Write block 496 len 36864 data: 2 3 4 4 3 3 3 3 3 17/30 Write block 65 len 36864 data: 7 6 5 4 5 4 3 3 3 18/30 Flush 19/30 Write block 296 len 16384 data: 3 4 4 4 20/30 Write block 140 len 24576 data: 3 3 3 2 3 3 21/30 Read block 8 len 36864 22/30 Read block 52 len 32768 23/30 Read block 73 len 8192 24/30 Flush 25/30 Write block 321 len 12288 data: 3 3 3 26/30 Read block 529 len 4096 27/30 Write block 429 len 8192 data: 2 2 28/30 Read block 443 len 24576 29/30 Read block 294 len 16384 30/30 Write block 514 len 12288 data: 6 6 5 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:14.450233041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 28 28 28 45 45 45 F F F 3 060-079 28 28 28 46 46 46 F F F 4 080-099 28 28 28 45 45 45 F F F 5 100-119 26 26 26 42 42 42 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 28 28 28 47 47 47 F F F 8 160-179 26 26 26 43 43 43 F F F 9 180-199 26 26 26 43 43 43 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 28 28 28 45 45 45 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 26 26 26 43 43 43 T T T 17 340-359 26 26 26 43 43 43 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 28 28 28 45 45 45 F F F 21 420-439 16 16 16 28 28 28 T T T 22 440-459 25 25 25 41 41 41 F F F 23 460-479 24 24 24 38 38 38 F F F 24 480-499 28 28 28 46 46 46 F F F 25 500-519 28 28 28 46 46 46 T T T 26 520-539 26 26 26 42 42 42 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 46 46 46 F F F Max gen: 28, Max flush: 47 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 20 20 20 33 33 33 F F F 2 040-059 28 28 28 45 45 45 F F F 3 060-079 28 28 28 46 46 46 F F F 4 080-099 28 28 28 45 45 45 F F F 5 100-119 26 26 26 42 42 42 F F F 6 120-139 19 19 19 32 32 32 F F F 7 140-159 28 28 28 47 47 47 F F F 8 160-179 26 26 26 43 43 43 F F F 9 180-199 26 26 26 43 43 43 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 28 28 28 45 45 45 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 26 26 26 43 43 43 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 28 28 28 45 45 45 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 25 25 25 41 41 41 F F F 23 460-479 24 24 24 38 38 38 F F F 24 480-499 28 28 28 46 46 46 F F F 25 500-519 29 29 29 48 48 48 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 46 46 46 F F F Max gen: 29, Max flush: 48 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.685614426Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.729899475Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.730629205Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.730665929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.730680818Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.730695705Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} {"msg":"Crucible 6115dbdc-847b-496a-9d53-a6a02086bcbf has session id: f187565b-25e5-4d69-ad75-b39cc65309a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.731310681Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Upstairs opts: Upstairs UUID: 6115dbdc-847b-496a-9d53-a6a02086bcbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.731358124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Crucible stats registered with UUID: 6115dbdc-847b-496a-9d53-a6a02086bcbf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.731377762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:28.73139194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.731915508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.731941324Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} {"msg":"6115dbdc-847b-496a-9d53-a6a02086bcbf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732015184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732034061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"0","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732064947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"0","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732083345Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"1","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732098423Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"1","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732112031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732126829Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732140927Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732163663Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732181371Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51924)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732215367Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:28.732306764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:28.732347688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38567)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732376744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:28.73240978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.732614501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.733162766Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"0","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.733191482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"1","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.733810587Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"1","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.733839063Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"0","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.734407005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:28.734501622Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.734490659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:53866)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.734620661Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.735695883Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736199904Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[0]R flush_numbers[0..12]: [45, 33, 45, 46, 45, 42, 32, 47, 43, 43, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736778514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[0]R generation[0..12]: [28, 20, 28, 28, 28, 26, 19, 28, 26, 26, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.73680365Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736823388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[1]R flush_numbers[0..12]: [45, 33, 45, 46, 45, 42, 32, 47, 43, 43, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736841445Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[1]R generation[0..12]: [28, 20, 28, 28, 28, 26, 19, 28, 26, 26, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736859203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736884709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[2]R flush_numbers[0..12]: [45, 33, 45, 46, 45, 42, 32, 47, 43, 43, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736902997Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[2]R generation[0..12]: [28, 20, 28, 28, 28, 26, 19, 28, 26, 26, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736920844Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736938612Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.73695541Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736972557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.736989385Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737006163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"":"downstairs","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.73702362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737038928Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"0","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"Wait for a query_work_queue command to finish before sending IO } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737061355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"1","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737078823Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"client":"2","":"downstairs","session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.73709626Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737111968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"6115dbdc-847b-496a-9d53-a6a02086bcbf is now active with session: f187565b-25e5-4d69-ad75-b39cc65309a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737128366Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737143834Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312,"session_id":"f187565b-25e5-4d69-ad75-b39cc65309a8"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737161851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} {"msg":"Activated sub_volume 6115dbdc-847b-496a-9d53-a6a02086bcbf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:38.737176569Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 180 len 8192 data: 3 2 02/30 Read block 147 len 20480 03/30 Write block 115 len 28672 data: 2 3 3 3 4 4 5 04/30 Read block 143 len 12288 05/30 Flush 06/30 Write block 119 len 32768 data: 5 5 6 5 3 2 3 3 07/30 Write block 95 len 32768 data: 5 5 5 5 6 5 4 3 08/30 Read block 130 len 24576 09/30 Flush 10/30 Write block 32 len 8192 data: 2 2 11/30 Write block 476 len 8192 data: 3 3 12/30 Write block 262 len 28672 data: 7 5 4 3 3 3 3 13/30 Write block 181 len 32768 data: 3 2 2 2 2 2 2 3 14/30 Flush 15/30 Write block 75 len 12288 data: 3 4 4 16/30 Read block 230 len 12288 17/30 Write block 520 len 28672 data: 5 5 5 4 4 5 4 18/30 Read block 475 len 24576 19/30 Read block 403 len 28672 20/30 Read block 430 len 8192 21/30 Read block 322 len 36864 22/30 Write block 412 len 16384 data: 3 4 4 3 23/30 Read block 53 len 24576 24/30 Flush 25/30 Write block 473 len 36864 data: 3 4 4 4 4 3 4 4 4 26/30 Write block 88 len 24576 data: 5 5 5 5 5 7 27/30 Read block 246 len 40960 28/30 Write block 39 len 36864 data: 4 5 6 7 6 5 4 3 4 29/30 Read block 164 len 12288 30/30 Write block 407 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:38.758441644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 30 30 30 51 51 51 T T T 2 040-059 28 28 28 45 45 45 T T T 3 060-079 30 30 30 52 52 52 F F F 4 080-099 30 30 30 50 50 50 T T T 5 100-119 30 30 30 50 50 50 F F F 6 120-139 30 30 30 50 50 50 F F F 7 140-159 28 28 28 47 47 47 F F F 8 160-179 26 26 26 43 43 43 F F F 9 180-199 30 30 30 51 51 51 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 30 30 30 51 51 51 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 26 26 26 43 43 43 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 30 30 30 52 52 52 T T T 21 420-439 29 29 29 48 48 48 F F F 22 440-459 25 25 25 41 41 41 F F F 23 460-479 30 30 30 51 51 51 T T T 24 480-499 28 28 28 46 46 46 T T T 25 500-519 29 29 29 48 48 48 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 46 46 46 F F F Max gen: 30, Max flush: 52 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 31 31 31 53 53 53 F F F 2 040-059 31 31 31 53 53 53 F F F 3 060-079 30 30 30 52 52 52 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 30 30 30 50 50 50 F F F 6 120-139 30 30 30 50 50 50 F F F 7 140-159 28 28 28 47 47 47 F F F 8 160-179 26 26 26 43 43 43 F F F 9 180-199 30 30 30 51 51 51 F F F 10 200-219 27 27 27 44 44 44 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 28 28 28 45 45 45 F F F 13 260-279 30 30 30 51 51 51 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 26 26 26 43 43 43 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 25 25 25 41 41 41 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 31 31 31 53 53 53 F F F 25 500-519 29 29 29 48 48 48 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 F F F 29 580-599 28 28 28 46 46 46 F F F Max gen: 31, Max flush: 53 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.011558194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.056247388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.057240041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.057289375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.057310572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.057339058Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} {"msg":"Crucible fb20fc1c-bca6-4c9c-a49e-eb1b424ba989 has session id: e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.057941455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Upstairs opts: Upstairs UUID: fb20fc1c-bca6-4c9c-a49e-eb1b424ba989, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.057983359Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Crucible stats registered with UUID: fb20fc1c-bca6-4c9c-a49e-eb1b424ba989","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058017244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:43.058032602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058509277Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058534243Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} {"msg":"fb20fc1c-bca6-4c9c-a49e-eb1b424ba989 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058595905Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058621441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"0","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058653577Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"0","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058668895Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"1","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058684073Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"1","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058698441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"2","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058723417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"2","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058751863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.05877644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058801347Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42618)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058826943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38127)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058852769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64714)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.058880156Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.059572071Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"2","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.059602366Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"0","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.060285683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"0","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.060330246Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"2","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.061033849Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.061072174Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.061286295Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"1","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.061793735Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"1","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[0]R flush_numbers[0..12]: [45, 53, 53, 52, 53, 50, 50, 47, 43, 51, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062317673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[0]R generation[0..12]: [28, 31, 31, 30, 31, 30, 30, 28, 26, 30, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062347609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062364226Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[1]R flush_numbers[0..12]: [45, 53, 53, 52, 53, 50, 50, 47, 43, 51, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062379024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[1]R generation[0..12]: [28, 31, 31, 30, 31, 30, 30, 28, 26, 30, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062393433Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062407991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[2]R flush_numbers[0..12]: [45, 53, 53, 52, 53, 50, 50, 47, 43, 51, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062422149Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[2]R generation[0..12]: [28, 31, 31, 30, 31, 30, 30, 28, 26, 30, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062436097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062450425Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062463773Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062477481Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062490879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:02:43.062504537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"":"downstairs","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062525095Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062537663Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"0","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.06255388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"1","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062568009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"client":"2","":"downstairs","session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062581807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062594325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"fb20fc1c-bca6-4c9c-a49e-eb1b424ba989 is now active with session: e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062607363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062619662Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320,"session_id":"e4d9bf4c-2e73-40b4-ba46-3a2a9d942e3e"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.06263209Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} {"msg":"Activated sub_volume fb20fc1c-bca6-4c9c-a49e-eb1b424ba989","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:43.062643738Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 20480 data: 4 3 3 3 3 02/30 Read block 124 len 40960 03/30 Write block 133 len 24576 data: 3 3 3 2 2 2 04/30 Write block 10 len 40960 data: 3 3 3 3 3 2 3 3 3 3 05/30 Read block 360 len 16384 06/30 Flush 07/30 Write block 103 len 12288 data: 5 5 4 08/30 Flush 09/30 Write block 445 len 28672 data: 4 4 4 4 4 4 3 10/30 Write block 451 len 36864 data: 4 2 2 2 2 2 2 2 2 11/30 Read block 291 len 40960 12/30 Write block 271 len 4096 data: 2 13/30 Flush 14/30 Write block 128 len 36864 data: 3 3 3 3 3 4 4 4 3 15/30 Write block 243 len 40960 data: 4 4 4 4 3 3 2 2 2 3 16/30 Write block 40 len 28672 data: 6 7 8 7 6 5 4 17/30 Write block 213 len 4096 data: 4 18/30 Read block 456 len 20480 19/30 Read block 433 len 20480 20/30 Read block 75 len 16384 21/30 Write block 130 len 16384 data: 4 4 4 5 22/30 Write block 190 len 8192 data: 4 4 23/30 Write block 561 len 40960 data: 7 7 6 6 7 7 6 5 4 3 24/30 Read block 545 len 24576 25/30 Write block 217 len 16384 data: 4 4 4 5 26/30 Read block 530 len 28672 27/30 Read block 191 len 40960 28/30 Write block 145 len 40960 data: 4 3 2 2 2 2 2 2 2 4 29/30 Write block 579 len 28672 data: 4 4 4 4 4 5 6 30/30 Write block 499 len 40960 data: 5 4 4 4 4 4 3 3 3 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:43.083660917Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 54 54 54 F F F 1 020-039 31 31 31 53 53 53 F F F 2 040-059 31 31 31 53 53 53 T T T 3 060-079 30 30 30 52 52 52 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 32 32 32 54 54 54 T T T 7 140-159 28 28 28 47 47 47 T T T 8 160-179 26 26 26 43 43 43 F F F 9 180-199 30 30 30 51 51 51 T T T 10 200-219 27 27 27 44 44 44 T T T 11 220-239 27 27 27 44 44 44 T T T 12 240-259 28 28 28 45 45 45 T T T 13 260-279 32 32 32 56 56 56 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 32 32 32 54 54 54 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 31 31 31 53 53 53 T T T 25 500-519 29 29 29 48 48 48 T T T 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 28 28 28 45 45 45 T T T 29 580-599 28 28 28 46 46 46 T T T Max gen: 32, Max flush: 56 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 54 54 54 F F F 1 020-039 31 31 31 53 53 53 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 30 30 30 52 52 52 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 33 33 33 57 57 57 F F F 7 140-159 33 33 33 57 57 57 F F F 8 160-179 26 26 26 43 43 43 F F F 9 180-199 33 33 33 57 57 57 F F F 10 200-219 33 33 33 57 57 57 F F F 11 220-239 33 33 33 57 57 57 F F F 12 240-259 33 33 33 57 57 57 F F F 13 260-279 32 32 32 56 56 56 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 32 32 32 54 54 54 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 33 33 33 57 57 57 F F F 25 500-519 33 33 33 57 57 57 F F F 26 520-539 30 30 30 52 52 52 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 33 33 33 57 57 57 F F F 29 580-599 33 33 33 57 57 57 F F F Max gen: 33, Max flush: 57 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.358061007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.402036519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.402829219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.402864375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.402886232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.402906439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} {"msg":"Crucible 091eb15b-5a29-4719-b465-e8576eb1b672 has session id: 3a7243e0-0853-483b-b9cc-0b5dc6230dfd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.403568208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Upstairs opts: Upstairs UUID: 091eb15b-5a29-4719-b465-e8576eb1b672, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.403601433Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Crucible stats registered with UUID: 091eb15b-5a29-4719-b465-e8576eb1b672","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.403617571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:57.403631129Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404260873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404290998Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} {"msg":"091eb15b-5a29-4719-b465-e8576eb1b672 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404362528Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404380536Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"0","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404409532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"0","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.40442443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404439658Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404453526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"2","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404468044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"2","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404484732Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404508148Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404533345Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:51525)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.40457106Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:57.404771522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50377)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404807877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:57.404826345Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:57.404846252Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.404939709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.405285072Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"0","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.40544449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"2","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.405853154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"0","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.405989295Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"2","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.406396129Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:57.406509643Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.407119594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41188)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.407230219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.408264397Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.408756609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[0]R flush_numbers[0..12]: [54, 53, 57, 52, 53, 55, 57, 57, 43, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409336849Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[0]R generation[0..12]: [32, 31, 33, 30, 31, 32, 33, 33, 26, 33, 33, 33]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409364046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409388572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[1]R flush_numbers[0..12]: [54, 53, 57, 52, 53, 55, 57, 57, 43, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.40940706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[1]R generation[0..12]: [32, 31, 33, 30, 31, 32, 33, 33, 26, 33, 33, 33]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409424787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409442805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[2]R flush_numbers[0..12]: [54, 53, 57, 52, 53, 55, 57, 57, 43, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409460452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[2]R generation[0..12]: [32, 31, 33, 30, 31, 32, 33, 33, 26, 33, 33, 33]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.4094779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409495978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409512785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409529903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409556999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409588215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"":"downstairs","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409608702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.40962432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"0","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409641627Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"1","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409658665Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"client":"2","":"downstairs","session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409678453Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.40969443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"091eb15b-5a29-4719-b465-e8576eb1b672 is now active with session: 3a7243e0-0853-483b-b9cc-0b5dc6230dfd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409710718Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409725946Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328,"session_id":"3a7243e0-0853-483b-b9cc-0b5dc6230dfd"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409741474Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} {"msg":"Activated sub_volume 091eb15b-5a29-4719-b465-e8576eb1b672","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:07.409756032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 20480 data: 6 6 6 6 6 02/30 Write block 155 len 16384 data: 4 5 5 4 03/30 Flush 04/30 Write block 130 len 12288 data: 5 5 5 05/30 Read block 347 len 8192 06/30 Write block 447 len 24576 data: 5 5 5 5 5 3 07/30 Write block 65 len 40960 data: 8 7 6 5 6 5 4 4 4 2 08/30 Write block 539 len 32768 data: 3 2 2 2 3 3 3 3 09/30 Write block 63 len 24576 data: 7 7 9 8 7 6 10/30 Write block 537 len 24576 data: 4 3 4 3 3 3 11/30 Read block 265 len 24576 12/30 Read block 182 len 4096 13/30 Write block 244 len 28672 data: 5 5 5 4 4 3 3 14/30 Write block 146 len 20480 data: 4 3 3 3 3 15/30 Read block 266 len 28672 16/30 Read block 235 len 8192 17/30 Read block 353 len 4096 18/30 Read block 559 len 36864 19/30 Read block 191 len 28672 20/30 Write block 159 len 8192 data: 3 3 21/30 Read block 44 len 12288 22/30 Write block 275 len 4096 data: 2 23/30 Write block 53 len 20480 data: 3 3 5 3 3 24/30 Flush 25/30 Write block 214 len 32768 data: 4 3 3 5 5 5 6 7 26/30 Write block 39 len 36864 data: 5 7 8 9 8 7 6 5 5 27/30 Read block 302 len 28672 28/30 Read block 251 len 4096 29/30 Write block 445 len 40960 data: 5 5 6 6 6 6 6 4 3 3 30/30 Read block 210 len 4096 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:07.430143788Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 54 54 54 F F F 1 020-039 31 31 31 53 53 53 T T T 2 040-059 34 34 34 59 59 59 T T T 3 060-079 34 34 34 59 59 59 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 34 34 34 59 59 59 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 34 34 34 59 59 59 F F F 9 180-199 33 33 33 57 57 57 F F F 10 200-219 33 33 33 57 57 57 T T T 11 220-239 33 33 33 57 57 57 T T T 12 240-259 34 34 34 59 59 59 F F F 13 260-279 34 34 34 59 59 59 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 32 32 32 54 54 54 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 34 34 34 59 59 59 T T T 23 460-479 31 31 31 53 53 53 F F F 24 480-499 33 33 33 57 57 57 F F F 25 500-519 33 33 33 57 57 57 F F F 26 520-539 34 34 34 59 59 59 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 33 33 33 57 57 57 F F F 29 580-599 33 33 33 57 57 57 F F F Max gen: 34, Max flush: 59 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 54 54 54 F F F 1 020-039 35 35 35 60 60 60 F F F 2 040-059 35 35 35 60 60 60 F F F 3 060-079 34 34 34 59 59 59 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 34 34 34 59 59 59 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 34 34 34 59 59 59 F F F 9 180-199 33 33 33 57 57 57 F F F 10 200-219 35 35 35 60 60 60 F F F 11 220-239 35 35 35 60 60 60 F F F 12 240-259 34 34 34 59 59 59 F F F 13 260-279 34 34 34 59 59 59 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 32 32 32 54 54 54 F F F 18 360-379 19 19 19 32 32 32 F F F 19 380-399 13 13 13 21 21 21 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 35 35 35 60 60 60 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 33 33 33 57 57 57 F F F 25 500-519 33 33 33 57 57 57 F F F 26 520-539 34 34 34 59 59 59 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 33 33 33 57 57 57 F F F 29 580-599 33 33 33 57 57 57 F F F Max gen: 35, Max flush: 60 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.677352855Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.723093504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.723886895Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.723929639Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.723946737Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.723965244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} {"msg":"Crucible 55ba523d-2d04-4bd5-bafe-fc2dd3871b3d has session id: ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.724566032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Upstairs opts: Upstairs UUID: 55ba523d-2d04-4bd5-bafe-fc2dd3871b3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.724601787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Crucible stats registered with UUID: 55ba523d-2d04-4bd5-bafe-fc2dd3871b3d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.724617195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:21.724630233Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725118366Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725150551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} {"msg":"55ba523d-2d04-4bd5-bafe-fc2dd3871b3d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725223021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725240349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"0","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725270675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"0","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725297891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"1","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725319358Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"1","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725333606Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"2","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725356413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"2","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.72538123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725406836Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725432822Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"ds_connection connected from Ok(127.0.0.1:38319)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725536798Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"ds_connection connected from Ok(127.0.0.1:50081)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725578372Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"ds_connection connected from Ok(127.0.0.1:58073)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.725610318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.726294364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"1","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.72632313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"0","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.726849668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"2","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.726998937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"1","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.727027313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"0","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.72741129Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"2","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.727677244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.727713909Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[0]R flush_numbers[0..12]: [54, 60, 60, 59, 58, 55, 59, 59, 59, 57, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728009968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[0]R generation[0..12]: [32, 35, 35, 34, 34, 32, 34, 34, 34, 33, 35, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728037204Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728053122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[1]R flush_numbers[0..12]: [54, 60, 60, 59, 58, 55, 59, 59, 59, 57, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.72806777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[1]R generation[0..12]: [32, 35, 35, 34, 34, 32, 34, 34, 34, 33, 35, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728082178Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728096596Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[2]R flush_numbers[0..12]: [54, 60, 60, 59, 58, 55, 59, 59, 59, 57, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728110584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[2]R generation[0..12]: [32, 35, 35, 34, 34, 32, 34, 34, 34, 33, 35, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728124502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.72813868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728152168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728165956Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:21.728184374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728205191Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"":"downstairs","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728219069Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728232437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"0","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728246445Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"1","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728260043Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"client":"2","":"downstairs","session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728273652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.72828612Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"55ba523d-2d04-4bd5-bafe-fc2dd3871b3d is now active with session: ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728299208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728311456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336,"session_id":"ed11b7d8-0bb8-4fe7-8a30-2c547d6e4473"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728323875Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} {"msg":"Activated sub_volume 55ba523d-2d04-4bd5-bafe-fc2dd3871b3d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:21.728335383Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 121 len 20480 data: 7 6 4 3 4 02/30 Write block 379 len 40960 data: 2 2 2 2 2 2 2 2 2 2 03/30 Write block 209 len 8192 data: 4 4 04/30 Flush 05/30 Write block 465 len 36864 data: 2 2 2 4 2 2 2 3 4 06/30 Read block 593 len 4096 07/30 Read block 158 len 12288 08/30 Write block 241 len 4096 data: 4 09/30 Flush 10/30 Write block 336 len 36864 data: 5 6 6 5 5 5 5 4 4 11/30 Flush 12/30 Write block 239 len 12288 data: 5 5 5 13/30 Write block 580 len 32768 data: 5 5 5 5 6 7 6 5 14/30 Write block 308 len 16384 data: 2 2 2 3 15/30 Write block 273 len 40960 data: 2 2 3 2 2 3 3 5 5 5 16/30 Write block 564 len 12288 data: 7 8 8 17/30 Read block 343 len 28672 18/30 Write block 493 len 20480 data: 3 4 4 3 4 19/30 Read block 223 len 32768 20/30 Write block 545 len 32768 data: 4 4 4 4 5 5 5 6 21/30 Read block 99 len 12288 22/30 Write block 14 len 32768 data: 4 3 4 4 4 4 3 2 23/30 Write block 288 len 36864 data: 3 3 4 5 5 4 3 3 4 24/30 Write block 256 len 40960 data: 5 5 5 7 7 7 8 6 5 4 25/30 Write block 351 len 24576 data: 3 4 5 4 3 4 26/30 Write block 388 len 12288 data: 3 2 2 27/30 Write block 342 len 16384 data: 6 5 5 4 28/30 Write block 583 len 4096 data: 6 29/30 Read block 389 len 16384 30/30 Write block 324 len 32768 data: 3 3 2 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:21.747215996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 54 54 54 T T T 1 020-039 35 35 35 60 60 60 T T T 2 040-059 35 35 35 60 60 60 F F F 3 060-079 34 34 34 59 59 59 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 36 36 36 61 61 61 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 34 34 34 59 59 59 F F F 9 180-199 33 33 33 57 57 57 F F F 10 200-219 36 36 36 61 61 61 F F F 11 220-239 35 35 35 60 60 60 T T T 12 240-259 36 36 36 62 62 62 T T T 13 260-279 34 34 34 59 59 59 T T T 14 280-299 28 28 28 47 47 47 T T T 15 300-319 26 26 26 42 42 42 T T T 16 320-339 36 36 36 63 63 63 T T T 17 340-359 36 36 36 63 63 63 T T T 18 360-379 36 36 36 61 61 61 F F F 19 380-399 36 36 36 61 61 61 T T T 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 35 35 35 60 60 60 F F F 23 460-479 36 36 36 62 62 62 F F F 24 480-499 33 33 33 57 57 57 T T T 25 500-519 33 33 33 57 57 57 F F F 26 520-539 34 34 34 59 59 59 F F F 27 540-559 34 34 34 59 59 59 T T T 28 560-579 33 33 33 57 57 57 T T T 29 580-599 33 33 33 57 57 57 T T T Max gen: 36, Max flush: 63 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 35 35 35 60 60 60 F F F 3 060-079 34 34 34 59 59 59 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 36 36 36 61 61 61 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 34 34 34 59 59 59 F F F 9 180-199 33 33 33 57 57 57 F F F 10 200-219 36 36 36 61 61 61 F F F 11 220-239 37 37 37 64 64 64 F F F 12 240-259 37 37 37 64 64 64 F F F 13 260-279 37 37 37 64 64 64 F F F 14 280-299 37 37 37 64 64 64 F F F 15 300-319 37 37 37 64 64 64 F F F 16 320-339 37 37 37 64 64 64 F F F 17 340-359 37 37 37 64 64 64 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 64 64 64 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 35 35 35 60 60 60 F F F 23 460-479 36 36 36 62 62 62 F F F 24 480-499 37 37 37 64 64 64 F F F 25 500-519 33 33 33 57 57 57 F F F 26 520-539 34 34 34 59 59 59 F F F 27 540-559 37 37 37 64 64 64 F F F 28 560-579 37 37 37 64 64 64 F F F 29 580-599 37 37 37 64 64 64 F F F Max gen: 37, Max flush: 64 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.040822311Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.085027971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.085752651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.085790406Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.085805724Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.085820862Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} {"msg":"Crucible cce3b8f3-bf42-4e0c-bcc5-2b401e9b6217 has session id: 0ed62060-49d7-4508-98a5-df59fd622302","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.086372366Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Upstairs opts: Upstairs UUID: cce3b8f3-bf42-4e0c-bcc5-2b401e9b6217, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.086422199Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Crucible stats registered with UUID: cce3b8f3-bf42-4e0c-bcc5-2b401e9b6217","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.086449666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:36.086468153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.086981722Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087012898Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} {"msg":"cce3b8f3-bf42-4e0c-bcc5-2b401e9b6217 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.08707636Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087094277Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"0","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087122463Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"0","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.08714354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"1","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087170076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"1","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087196563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"2","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087225949Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"2","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087251795Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087279401Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087301029Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"ds_connection connected from Ok(127.0.0.1:50993)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087351042Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"ds_connection connected from Ok(127.0.0.1:49320)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087386507Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"ds_connection connected from Ok(127.0.0.1:56920)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.087416633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.088119896Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"2","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.0881586Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"0","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.088686218Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"0","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.08881585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"2","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.08932901Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.089408479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.08968679Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"1","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090173943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"1","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[0]R flush_numbers[0..12]: [64, 64, 60, 59, 58, 55, 61, 59, 59, 57, 61, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090680064Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[0]R generation[0..12]: [37, 37, 35, 34, 34, 32, 36, 34, 34, 33, 36, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090701901Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090717759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[1]R flush_numbers[0..12]: [64, 64, 60, 59, 58, 55, 61, 59, 59, 57, 61, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090736976Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[1]R generation[0..12]: [37, 37, 35, 34, 34, 32, 36, 34, 34, 33, 36, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090751744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090766172Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[2]R flush_numbers[0..12]: [64, 64, 60, 59, 58, 55, 61, 59, 59, 57, 61, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.09078031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[2]R generation[0..12]: [37, 37, 35, 34, 34, 32, 36, 34, 34, 33, 36, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090797618Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090823344Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090849681Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090872437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090886985Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090901303Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"":"downstairs","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090915741Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.09092826Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"0","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090942268Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"1","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090955746Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"client":"2","":"downstairs","session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090975603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.090991111Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"cce3b8f3-bf42-4e0c-bcc5-2b401e9b6217 is now active with session: 0ed62060-49d7-4508-98a5-df59fd622302","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.091004759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.091017427Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344,"session_id":"0ed62060-49d7-4508-98a5-df59fd622302"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.091030016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} {"msg":"Activated sub_volume cce3b8f3-bf42-4e0c-bcc5-2b401e9b6217","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:36.091041714Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 43 len 24576 data: 9 8 7 6 6 3 02/30 Write block 151 len 36864 data: 3 3 3 5 5 6 6 5 4 03/30 Read block 123 len 24576 04/30 Read block 110 len 16384 05/30 Write block 447 len 28672 data: 7 7 7 7 7 5 4 06/30 Write block 576 len 20480 data: 2 2 4 5 6 07/30 Read block 227 len 8192 08/30 Write block 196 len 32768 data: 2 2 3 3 3 3 3 3 09/30 Read block 104 len 28672 10/30 Read block 293 len 20480 11/30 Write block 53 len 20480 data: 4 4 6 4 4 12/30 Write block 61 len 16384 data: 7 8 8 8 13/30 Write block 155 len 36864 data: 6 7 7 6 5 4 3 5 5 14/30 Write block 256 len 12288 data: 6 6 6 15/30 Flush 16/30 Write block 206 len 40960 data: 4 4 4 5 5 3 3 5 5 4 17/30 Write block 106 len 24576 data: 4 4 5 4 4 3 18/30 Write block 84 len 36864 data: 4 4 5 5 7 7 7 7 7 19/30 Read block 265 len 4096 20/30 Flush 21/30 Write block 156 len 40960 data: 8 8 7 6 5 4 6 6 6 5 22/30 Read block 220 len 8192 23/30 Read block 582 len 16384 24/30 Read block 284 len 24576 25/30 Read block 349 len 28672 26/30 Write block 135 len 4096 data: 5 27/30 Write block 492 len 24576 data: 5 4 5 5 4 5 28/30 Write block 126 len 8192 data: 4 3 29/30 Write block 212 len 28672 data: 4 6 6 5 4 6 6 30/30 Read block 392 len 20480 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 10 9 8 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:36.111160897Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 38 38 38 65 65 65 F F F 3 060-079 38 38 38 65 65 65 F F F 4 080-099 38 38 38 66 66 66 F F F 5 100-119 38 38 38 66 66 66 F F F 6 120-139 36 36 36 61 61 61 T T T 7 140-159 38 38 38 65 65 65 T T T 8 160-179 38 38 38 65 65 65 T T T 9 180-199 38 38 38 65 65 65 F F F 10 200-219 38 38 38 66 66 66 T T T 11 220-239 37 37 37 64 64 64 F F F 12 240-259 38 38 38 65 65 65 F F F 13 260-279 37 37 37 64 64 64 F F F 14 280-299 37 37 37 64 64 64 F F F 15 300-319 37 37 37 64 64 64 F F F 16 320-339 37 37 37 64 64 64 F F F 17 340-359 37 37 37 64 64 64 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 64 64 64 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 38 38 38 65 65 65 F F F 23 460-479 36 36 36 62 62 62 F F F 24 480-499 37 37 37 64 64 64 T T T 25 500-519 33 33 33 57 57 57 F F F 26 520-539 34 34 34 59 59 59 F F F 27 540-559 37 37 37 64 64 64 F F F 28 560-579 38 38 38 65 65 65 F F F 29 580-599 38 38 38 65 65 65 F F F Max gen: 38, Max flush: 66 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 38 38 38 65 65 65 F F F 3 060-079 38 38 38 65 65 65 F F F 4 080-099 38 38 38 66 66 66 F F F 5 100-119 38 38 38 66 66 66 F F F 6 120-139 39 39 39 67 67 67 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 39 39 39 67 67 67 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 39 39 39 67 67 67 F F F 11 220-239 37 37 37 64 64 64 F F F 12 240-259 38 38 38 65 65 65 F F F 13 260-279 37 37 37 64 64 64 F F F 14 280-299 37 37 37 64 64 64 F F F 15 300-319 37 37 37 64 64 64 F F F 16 320-339 37 37 37 64 64 64 F F F 17 340-359 37 37 37 64 64 64 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 64 64 64 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 38 38 38 65 65 65 F F F 23 460-479 36 36 36 62 62 62 F F F 24 480-499 39 39 39 67 67 67 F F F 25 500-519 33 33 33 57 57 57 F F F 26 520-539 34 34 34 59 59 59 F F F 27 540-559 37 37 37 64 64 64 F F F 28 560-579 38 38 38 65 65 65 F F F 29 580-599 38 38 38 65 65 65 F F F Max gen: 39, Max flush: 67 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.359788644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.40475476Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.405806805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.40584413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.405861628Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.405879745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} {"msg":"Crucible f9aba1a0-f39c-498f-81d5-6bf9679df90a has session id: 1ba9732f-bab6-4663-a6a0-2d35420fc2e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.406584318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Upstairs opts: Upstairs UUID: f9aba1a0-f39c-498f-81d5-6bf9679df90a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.406627112Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Crucible stats registered with UUID: f9aba1a0-f39c-498f-81d5-6bf9679df90a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.40664666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:40.406664457Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.40715074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407176767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} {"msg":"f9aba1a0-f39c-498f-81d5-6bf9679df90a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407271703Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407289611Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"0","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407318907Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"0","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407342684Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"1","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407380899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"1","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407413274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"2","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.40744344Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"2","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407473706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407507661Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407544776Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:38055)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407577911Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:34211)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407606417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:56139)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.407638573Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.408368093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"1","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.408404158Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"0","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.409089524Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"1","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.409128598Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"0","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.40976984Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.409868126Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"2","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.409934587Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.410460525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"2","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[0]R flush_numbers[0..12]: [64, 64, 65, 65, 66, 66, 67, 67, 67, 65, 67, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.410968805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[0]R generation[0..12]: [37, 37, 38, 38, 38, 38, 39, 39, 39, 38, 39, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.410999511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411023887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[1]R flush_numbers[0..12]: [64, 64, 65, 65, 66, 66, 67, 67, 67, 65, 67, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411054763Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[1]R generation[0..12]: [37, 37, 38, 38, 38, 38, 39, 39, 39, 38, 39, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411088218Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411122444Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[2]R flush_numbers[0..12]: [64, 64, 65, 65, 66, 66, 67, 67, 67, 65, 67, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411143011Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[2]R generation[0..12]: [37, 37, 38, 38, 38, 38, 39, 39, 39, 38, 39, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411162298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411193004Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.4112262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411256785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411279052Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411305899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pidWait for a query_work_queue command to finish before sending IO ":1352,"":"downstairs","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411344523Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411372899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"0","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411395166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"1","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411412294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"client":"2","":"downstairs","session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411436571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411464097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"f9aba1a0-f39c-498f-81d5-6bf9679df90a is now active with session: 1ba9732f-bab6-4663-a6a0-2d35420fc2e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411489813Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.41151719Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352,"session_id":"1ba9732f-bab6-4663-a6a0-2d35420fc2e6"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411547386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} {"msg":"Activated sub_volume f9aba1a0-f39c-498f-81d5-6bf9679df90a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.411574822Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 590 len 32768 data: 5 3 3 3 4 3 3 3 02/30 Read block 153 len 24576 03/30 Flush 04/30 Write block 305 len 16384 data: 3 2 2 3 05/30 Write block 285 len 20480 data: 3 4 4 4 4 06/30 Read block 532 len 12288 07/30 Read block 105 len 24576 08/30 Write block 440 len 20480 data: 3 4 4 4 3 09/30 Read block 517 len 8192 10/30 Write block 236 len 32768 data: 5 7 6 6 6 6 3 5 11/30 Write block 224 len 20480 data: 6 5 6 5 5 12/30 Write block 44 len 36864 data: 9 8 7 7 4 2 2 3 3 13/30 Read block 356 len 4096 14/30 Write block 493 len 16384 data: 5 6 6 5 15/30 Read block 170 len 16384 16/30 Read block 356 len 8192 17/30 Write block 459 len 24576 data: 3 2 2 2 2 2 18/30 Write block 458 len 8192 data: 3 4 19/30 Write block 344 len 28672 data: 6 5 4 3 3 3 3 20/30 Flush 21/30 Write block 240 len 4096 data: 7 22/30 Read block 358 len 28672 23/30 Read block 110 len 12288 24/30 Read block 222 len 12288 25/30 Read block 528 len 40960 26/30 Read block 404 len 12288 27/30 Write block 90 len 28672 data: 8 8 8 8 7 6 6 28/30 Write block 524 len 20480 data: 5 6 5 3 4 29/30 Read block 80 len 4096 30/30 Read block 496 len 28672 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 9 10 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:40.432226231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 40 40 40 69 69 69 F F F 3 060-079 38 38 38 65 65 65 F F F 4 080-099 38 38 38 66 66 66 T T T 5 100-119 38 38 38 66 66 66 F F F 6 120-139 39 39 39 67 67 67 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 39 39 39 67 67 67 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 39 39 39 67 67 67 F F F 11 220-239 40 40 40 69 69 69 F F F 12 240-259 40 40 40 69 69 69 T T T 13 260-279 37 37 37 64 64 64 F F F 14 280-299 40 40 40 69 69 69 F F F 15 300-319 40 40 40 69 69 69 F F F 16 320-339 37 37 37 64 64 64 F F F 17 340-359 40 40 40 69 69 69 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 64 64 64 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 40 40 40 69 69 69 F F F 23 460-479 40 40 40 69 69 69 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 33 33 33 57 57 57 F F F 26 520-539 34 34 34 59 59 59 T T T 27 540-559 37 37 37 64 64 64 F F F 28 560-579 38 38 38 65 65 65 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 40, Max flush: 69 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 40 40 40 69 69 69 F F F 3 060-079 38 38 38 65 65 65 F F F 4 080-099 41 41 41 70 70 70 F F F 5 100-119 38 38 38 66 66 66 F F F 6 120-139 39 39 39 67 67 67 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 39 39 39 67 67 67 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 39 39 39 67 67 67 F F F 11 220-239 40 40 40 69 69 69 F F F 12 240-259 41 41 41 70 70 70 F F F 13 260-279 37 37 37 64 64 64 F F F 14 280-299 40 40 40 69 69 69 F F F 15 300-319 40 40 40 69 69 69 F F F 16 320-339 37 37 37 64 64 64 F F F 17 340-359 40 40 40 69 69 69 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 64 64 64 F F F 20 400-419 31 31 31 53 53 53 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 40 40 40 69 69 69 F F F 23 460-479 40 40 40 69 69 69 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 33 33 33 57 57 57 F F F 26 520-539 41 41 41 70 70 70 F F F 27 540-559 37 37 37 64 64 64 F F F 28 560-579 38 38 38 65 65 65 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 41, Max flush: 70 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.667298803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.711252428Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.711982157Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.712027261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.712044089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.712059407Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} {"msg":"Crucible 382bff2b-d4af-4ef1-a956-3a7c852ecb68 has session id: 00111441-5277-4b3b-abbb-0a8993869fab","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.712628978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Upstairs opts: Upstairs UUID: 382bff2b-d4af-4ef1-a956-3a7c852ecb68, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.712668253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Crucible stats registered with UUID: 382bff2b-d4af-4ef1-a956-3a7c852ecb68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.712685391Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:44.712703078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713155896Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713180102Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} {"msg":"382bff2b-d4af-4ef1-a956-3a7c852ecb68 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713227886Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713254822Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"0","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713284098Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"0","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713300066Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713322163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.71334847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"2","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713376096Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"2","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713402282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713428568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.71348752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"ds_connection connected from Ok(127.0.0.1:49893)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713525565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"ds_connection connected from Ok(127.0.0.1:53472)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713553121Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:44.713577938Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:44.713610723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:44.713645389Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.713694502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.714233128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"2","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.714262914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"0","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.714827526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"0","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.714954298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"2","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.715554776Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:44.715588751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.714818093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"ds_connection connected from Ok(127.0.0.1:42215)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.714896852Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.716014198Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.716614805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[0]R flush_numbers[0..12]: [64, 64, 69, 65, 70, 66, 67, 67, 67, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717209304Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[0]R generation[0..12]: [37, 37, 40, 38, 41, 38, 39, 39, 39, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717246068Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717273505Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[1]R flush_numbers[0..12]: [64, 64, 69, 65, 70, 66, 67, 67, 67, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.7173042Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[1]R generation[0..12]: [37, 37, 40, 38, 41, 38, 39, 39, 39, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717335996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717370122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[2]R flush_numbers[0..12]: [64, 64, 69, 65, 70, 66, 67, 67, 67, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717399697Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[2]R generation[0..12]: [37, 37, 40, 38, 41, 38, 39, 39, 39, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717427554Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717459649Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717491415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717513862Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717536328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"All extents match"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717570614Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"":"downstairs","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.71760295Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717627266Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"0","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717650983Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"1","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717681059Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"client":"2","":"downstairs","session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717713504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717736341Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"382bff2b-d4af-4ef1-a956-3a7c852ecb68 is now active with session: 00111441-5277-4b3b-abbb-0a8993869fab","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717756758Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717783485Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360,"session_id":"00111441-5277-4b3b-abbb-0a8993869fab"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717812981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} {"msg":"Activated sub_volume 382bff2b-d4af-4ef1-a956-3a7c852ecb68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.717838417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 471 len 28672 data: 3 4 5 5 5 5 5 03/30 Read block 310 len 28672 04/30 Read block 317 len 24576 05/30 Read block 389 len 24576 06/30 Read block 398 len 16384 07/30 Flush 08/30 Write block 561 len 20480 data: 8 8 7 8 9 09/30 Read block 47 len 12288 10/30 Flush 11/30 Write block 312 len 32768 data: 3 3 3 3 3 2 3 3 12/30 Write block 510 len 24576 data: 3 3 4 5 7 7 13/30 Read block 545 len 32768 14/30 Read block 438 len 8192 15/30 Write block 449 len 40960 data: 8 8 8 6 5 4 3 3 3 4 16/30 Write block 466 len 32768 data: 3 3 5 3 3 4 5 6 17/30 Write block 278 len 24576 data: 4 4 6 6 6 5 18/30 Write block 169 len 28672 data: 5 5 5 4 4 3 2 19/30 Flush 20/30 Write block 76 len 24576 data: 5 5 5 5 5 6 21/30 Read block 488 len 40960 22/30 Write block 336 len 20480 data: 6 7 7 6 6 23/30 Write block 506 len 4096 data: 4 24/30 Read block 143 len 36864 25/30 Read block 453 len 32768 26/30 Flush 27/30 Write block 97 len 36864 data: 6 6 7 6 5 4 6 6 5 28/30 Read block 105 len 4096 29/30 Read block 216 len 36864 30/30 Write block 396 len 32768 data: 4 3 3 3 4 4 4 5 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:54.739490499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 40 40 40 69 69 69 F F F 3 060-079 42 42 42 75 75 75 F F F 4 080-099 42 42 42 75 75 75 T T T 5 100-119 38 38 38 66 66 66 T T T 6 120-139 39 39 39 67 67 67 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 42 42 42 74 74 74 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 39 39 39 67 67 67 F F F 11 220-239 40 40 40 69 69 69 F F F 12 240-259 41 41 41 70 70 70 F F F 13 260-279 42 42 42 74 74 74 F F F 14 280-299 42 42 42 74 74 74 F F F 15 300-319 42 42 42 74 74 74 F F F 16 320-339 42 42 42 75 75 75 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 64 64 64 T T T 20 400-419 31 31 31 53 53 53 T T T 21 420-439 29 29 29 48 48 48 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 74 74 74 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 42 42 42 75 75 75 F F F 26 520-539 41 41 41 70 70 70 F F F 27 540-559 37 37 37 64 64 64 F F F 28 560-579 42 42 42 73 73 73 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 42, Max flush: 75 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 40 40 40 69 69 69 F F F 3 060-079 42 42 42 75 75 75 F F F 4 080-099 43 43 43 76 76 76 F F F 5 100-119 43 43 43 76 76 76 F F F 6 120-139 39 39 39 67 67 67 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 42 42 42 74 74 74 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 39 39 39 67 67 67 F F F 11 220-239 40 40 40 69 69 69 F F F 12 240-259 41 41 41 70 70 70 F F F 13 260-279 42 42 42 74 74 74 F F F 14 280-299 42 42 42 74 74 74 F F F 15 300-319 42 42 42 74 74 74 F F F 16 320-339 42 42 42 75 75 75 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 43 43 43 76 76 76 F F F 20 400-419 43 43 43 76 76 76 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 74 74 74 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 42 42 42 75 75 75 F F F 26 520-539 41 41 41 70 70 70 F F F 27 540-559 37 37 37 64 64 64 F F F 28 560-579 42 42 42 73 73 73 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 43, Max flush: 76 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.981352726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:59.024649011Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.025115777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.056271372Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.056948869Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.056991073Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.057027568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.057047605Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} {"msg":"Crucible 8fb3e7dd-d223-4fc0-a18a-493247fb1677 has session id: 48382a9c-adc8-44ce-a9f1-65961bd2449c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.057571463Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Upstairs opts: Upstairs UUID: 8fb3e7dd-d223-4fc0-a18a-493247fb1677, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.057615817Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Crucible stats registered with UUID: 8fb3e7dd-d223-4fc0-a18a-493247fb1677","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.057640183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:59.057661491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058141944Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.05817465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} {"msg":"8fb3e7dd-d223-4fc0-a18a-493247fb1677 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058219754Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058242991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"0","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058278286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"0","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058296523Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"1","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.05831565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"1","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058341897Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"2","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058373693Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"2","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058404638Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058435404Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.05846099Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38432)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058487467Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50152)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058515423Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61524)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.058551508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.059237104Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"2","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.059465362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"1","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.059640968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"0","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.060007598Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"2","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.060192712Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"1","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.060580599Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"0","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.060617174Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.060831235Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[0]R flush_numbers[0..12]: [64, 64, 69, 75, 76, 76, 67, 67, 74, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061185726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[0]R generation[0..12]: [37, 37, 40, 42, 43, 43, 39, 39, 42, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061214402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061234569Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[1]R flush_numbers[0..12]: [64, 64, 69, 75, 76, 76, 67, 67, 74, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061260285Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[1]R generation[0..12]: [37, 37, 40, 42, 43, 43, 39, 39, 42, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061287752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061312068Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[2]R flush_numbers[0..12]: [64, 64, 69, 75, 76, 76, 67, 67, 74, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061327406Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[2]R generation[0..12]: [37, 37, 40, 42, 43, 43, 39, 39, 42, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061350703Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061377929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061408295Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061424743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.0614418Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:59.061464867Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"":"downstairs","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061489844Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061513741Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"0","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061530568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"1","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061544366Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"client":"2","":"downstairs","session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061561514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061582991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"8fb3e7dd-d223-4fc0-a18a-493247fb1677 is now active with session: 48382a9c-adc8-44ce-a9f1-65961bd2449c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061608028Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061622846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368,"session_id":"48382a9c-adc8-44ce-a9f1-65961bd2449c"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.061641573Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} {"msg":"Activated sub_volume 8fb3e7dd-d223-4fc0-a18a-493247fb1677","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:59.06166174Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 8192 data: 9 10 02/30 Write block 56 len 20480 data: 5 5 5 6 7 03/30 Read block 227 len 36864 04/30 Read block 592 len 16384 05/30 Read block 521 len 16384 06/30 Read block 265 len 16384 07/30 Write block 384 len 32768 data: 3 3 3 3 4 3 3 2 08/30 Read block 402 len 16384 09/30 Read block 530 len 8192 10/30 Write block 469 len 20480 data: 4 4 5 6 7 11/30 Read block 231 len 24576 12/30 Write block 99 len 12288 data: 8 7 6 13/30 Write block 124 len 24576 data: 4 5 5 4 4 4 14/30 Write block 447 len 8192 data: 8 8 15/30 Read block 537 len 20480 16/30 Read block 143 len 40960 17/30 Read block 259 len 12288 18/30 Read block 570 len 36864 19/30 Flush 20/30 Write block 536 len 16384 data: 5 5 4 5 21/30 Read block 546 len 12288 22/30 Read block 57 len 20480 23/30 Read block 154 len 24576 24/30 Write block 568 len 16384 data: 6 5 4 3 25/30 Write block 242 len 40960 data: 4 6 6 6 6 5 5 4 4 3 26/30 Read block 424 len 20480 27/30 Read block 325 len 16384 28/30 Write block 539 len 8192 data: 6 4 29/30 Read block 26 len 16384 30/30 Read block 97 len 28672 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 10 10 11 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:59.082471768Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 44 44 44 77 77 77 F F F 3 060-079 44 44 44 77 77 77 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 42 42 42 74 74 74 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 39 39 39 67 67 67 F F F 11 220-239 40 40 40 69 69 69 F F F 12 240-259 41 41 41 70 70 70 T T T 13 260-279 42 42 42 74 74 74 F F F 14 280-299 42 42 42 74 74 74 F F F 15 300-319 42 42 42 74 74 74 F F F 16 320-339 42 42 42 75 75 75 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 43 43 43 76 76 76 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 44 44 44 77 77 77 F F F 23 460-479 44 44 44 77 77 77 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 42 42 42 75 75 75 F F F 26 520-539 41 41 41 70 70 70 T T T 27 540-559 37 37 37 64 64 64 T T T 28 560-579 42 42 42 73 73 73 T T T 29 580-599 40 40 40 68 68 68 F F F Max gen: 44, Max flush: 77 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 44 44 44 77 77 77 F F F 3 060-079 44 44 44 77 77 77 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 39 39 39 67 67 67 F F F 8 160-179 42 42 42 74 74 74 F F F 9 180-199 38 38 38 65 65 65 F F F 10 200-219 39 39 39 67 67 67 F F F 11 220-239 40 40 40 69 69 69 F F F 12 240-259 45 45 45 78 78 78 F F F 13 260-279 42 42 42 74 74 74 F F F 14 280-299 42 42 42 74 74 74 F F F 15 300-319 42 42 42 74 74 74 F F F 16 320-339 42 42 42 75 75 75 F F F 17 340-359 42 42 42 75 75 75 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 43 43 43 76 76 76 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 44 44 44 77 77 77 F F F 23 460-479 44 44 44 77 77 77 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 42 42 42 75 75 75 F F F 26 520-539 45 45 45 78 78 78 F F F 27 540-559 45 45 45 78 78 78 F F F 28 560-579 45 45 45 78 78 78 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 45, Max flush: 78 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.35091549Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.395978532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.396840883Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.396890936Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.396912044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.39693481Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} {"msg":"Crucible 83bd65fb-270a-4b09-b18d-bc8de2b1a92d has session id: d05b64ad-78bf-4ce7-80ff-4c7be15f55ce","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.397503172Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Upstairs opts: Upstairs UUID: 83bd65fb-270a-4b09-b18d-bc8de2b1a92d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.397550666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Crucible stats registered with UUID: 83bd65fb-270a-4b09-b18d-bc8de2b1a92d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.397580162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:03.397600309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398202296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398236182Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} {"msg":"83bd65fb-270a-4b09-b18d-bc8de2b1a92d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398288304Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.39831632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"0","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398360744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"0","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.39839076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"1","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398421176Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"1","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398449222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"2","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398475878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"2","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398586813Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398637226Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398723784Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:56182)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.39875378Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:56970)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398777147Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:33894)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.398793675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.399298935Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"2","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.39933654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"0","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.399825353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"1","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.399935608Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"2","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.399957625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"0","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.400446187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"1","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.400630152Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.400655049Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[0]R flush_numbers[0..12]: [64, 64, 77, 77, 77, 77, 77, 67, 74, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.400971575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[0]R generation[0..12]: [37, 37, 44, 44, 44, 44, 44, 39, 42, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.400992122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.40100744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[1]R flush_numbers[0..12]: [64, 64, 77, 77, 77, 77, 77, 67, 74, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401022168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[1]R generation[0..12]: [37, 37, 44, 44, 44, 44, 44, 39, 42, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401039786Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401054554Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[2]R flush_numbers[0..12]: [64, 64, 77, 77, 77, 77, 77, 67, 74, 65, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401069092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[2]R generation[0..12]: [37, 37, 44, 44, 44, 44, 44, 39, 42, 38, 39, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.40108329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401097998Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401111676Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401125494Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401138892Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.40115333Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"":"downstairs","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401167558Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401186456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"0","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401200564Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"1","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401214352Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"client":"2","":"downstairs","session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.4012282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401241188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"83bd65fb-270a-4b09-b18d-bc8de2b1a92d is now active with session: d05b64ad-78bf-4ce7-80ff-4c7be15f55ce","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401256306Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401269024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376,"session_id":"d05b64ad-78bf-4ce7-80ff-4c7be15f55ce"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401281903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} {"msg":"Activated sub_volume 83bd65fb-270a-4b09-b18d-bc8de2b1a92d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.401293851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 10 len 28672 data: 4 4 4 4 5 4 5 02/30 Write block 424 len 16384 data: 4 4 3 3 03/30 Write block 147 len 20480 data: 4 4 4 4 4 04/30 Write block 248 len 12288 data: 6 5 5 05/30 Read block 103 len 20480 06/30 Read block 212 len 36864 07/30 Read block 329 len 24576 08/30 Write block 407 len 36864 data: 4 3 3 3 3 4 5 5 4 09/30 Write block 145 len 36864 data: 5 5 5 5 5 5 5 4 4 10/30 Write block 418 len 28672 data: 5 5 5 5 5 4 5 11/30 Write block 199 len 16384 data: 4 4 4 4 12/30 Write block 43 len 4096 data: 10 13/30 Write block 287 len 16384 data: 5 5 5 5 14/30 Read block 176 len 12288 15/30 Write block 213 len 32768 data: 7 7 6 5 7 7 6 7 16/30 Read block 508 len 8192 17/30 Read block 378 len 36864 18/30 Read block 123 len 32768 19/30 Write block 427 len 32768 data: 4 3 3 3 2 2 2 2 20/30 Write block 17 len 12288 data: 5 5 5 21/30 Read block 316 len 36864 22/30 Read block 39 len 24576 23/30 Read block 499 len 32768 24/30 Write block 510 len 24576 data: 4 4 5 6 8 8 25/30 Read block 156 len 28672 26/30 Write block 351 len 32768 data: 4 5 6 5 4 5 4 4 27/30 Write block 322 len 32768 data: 4 4 4 4 3 3 3 3 28/30 Write block 404 len 12288 data: 4 4 3 29/30 Read block 518 len 32768 30/30 Read block 458 len 8192 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:03.421276622Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 64 64 64 T T T 1 020-039 37 37 37 64 64 64 F F F 2 040-059 44 44 44 77 77 77 T T T 3 060-079 44 44 44 77 77 77 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 39 39 39 67 67 67 T T T 8 160-179 42 42 42 74 74 74 F F F 9 180-199 38 38 38 65 65 65 T T T 10 200-219 39 39 39 67 67 67 T T T 11 220-239 40 40 40 69 69 69 T T T 12 240-259 45 45 45 78 78 78 T T T 13 260-279 42 42 42 74 74 74 F F F 14 280-299 42 42 42 74 74 74 T T T 15 300-319 42 42 42 74 74 74 F F F 16 320-339 42 42 42 75 75 75 T T T 17 340-359 42 42 42 75 75 75 T T T 18 360-379 36 36 36 61 61 61 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 43 43 43 76 76 76 T T T 21 420-439 29 29 29 48 48 48 T T T 22 440-459 44 44 44 77 77 77 F F F 23 460-479 44 44 44 77 77 77 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 42 42 42 75 75 75 T T T 26 520-539 45 45 45 78 78 78 F F F 27 540-559 45 45 45 78 78 78 F F F 28 560-579 45 45 45 78 78 78 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 45, Max flush: 78 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 79 79 79 F F F 1 020-039 37 37 37 64 64 64 F F F 2 040-059 46 46 46 79 79 79 F F F 3 060-079 44 44 44 77 77 77 F F F 4 080-099 44 44 44 77 77 77 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 46 46 46 79 79 79 F F F 8 160-179 42 42 42 74 74 74 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 46 46 46 79 79 79 F F F 11 220-239 46 46 46 79 79 79 F F F 12 240-259 46 46 46 79 79 79 F F F 13 260-279 42 42 42 74 74 74 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 42 42 42 74 74 74 F F F 16 320-339 46 46 46 79 79 79 F F F 17 340-359 46 46 46 79 79 79 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 46 46 46 79 79 79 F F F 22 440-459 44 44 44 77 77 77 F F F 23 460-479 44 44 44 77 77 77 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 46 46 46 79 79 79 F F F 26 520-539 45 45 45 78 78 78 F F F 27 540-559 45 45 45 78 78 78 F F F 28 560-579 45 45 45 78 78 78 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 46, Max flush: 79 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.714111545Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.759352893Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.760238081Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.760283285Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.760301452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.76031995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} {"msg":"Crucible 76492ea5-f3ad-47f7-a17d-32682a2dc1a9 has session id: 095903dc-2584-4ee8-bcf8-c6e50d61af62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.760837859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Upstairs opts: Upstairs UUID: 76492ea5-f3ad-47f7-a17d-32682a2dc1a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.760966981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Crucible stats registered with UUID: 76492ea5-f3ad-47f7-a17d-32682a2dc1a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761003986Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:17.761031002Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761476721Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761505757Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} {"msg":"76492ea5-f3ad-47f7-a17d-32682a2dc1a9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761551951Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761568508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"0","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761597204Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"0","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761611792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"1","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.76162669Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"1","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761646717Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"2","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761675603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"2","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.76170287Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761730146Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761761432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"ds_connection connected from Ok(127.0.0.1:61478)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761816534Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"ds_connection connected from Ok(127.0.0.1:60027)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761860378Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"ds_connection connected from Ok(127.0.0.1:36122)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.761895673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.762832424Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"0","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.762880218Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"1","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.762966146Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"2","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.763511891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"1","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.763540487Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"0","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.763803311Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"2","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764127756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764213154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[0]R flush_numbers[0..12]: [79, 64, 79, 77, 77, 77, 77, 79, 74, 79, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764325249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[0]R generation[0..12]: [46, 37, 46, 44, 44, 44, 44, 46, 42, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764345636Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764361074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[1]R flush_numbers[0..12]: [79, 64, 79, 77, 77, 77, 77, 79, 74, 79, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764375432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[1]R generation[0..12]: [46, 37, 46, 44, 44, 44, 44, 46, 42, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.76438932Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764403948Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[2]R flush_numbers[0..12]: [79, 64, 79, 77, 77, 77, 77, 79, 74, 79, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764418096Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[2]R generation[0..12]: [46, 37, 46, 44, 44, 44, 44, 46, 42, 46, 46, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764432114Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764449892Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Max found gen is 47","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.76446377Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Generation requested: 48 >= found:47","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764477518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:17.764491036Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764508634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"":"downstairs","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764522292Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.76453455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"0","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764548328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"1","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764562166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"client":"2","":"downstairs","session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764576184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764588863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"76492ea5-f3ad-47f7-a17d-32682a2dc1a9 is now active with session: 095903dc-2584-4ee8-bcf8-c6e50d61af62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764601951Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764614309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384,"session_id":"095903dc-2584-4ee8-bcf8-c6e50d61af62"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764626937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} {"msg":"Activated sub_volume 76492ea5-f3ad-47f7-a17d-32682a2dc1a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.764638356Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 371 len 12288 data: 3 3 3 02/30 Flush 03/30 Write block 273 len 4096 data: 3 04/30 Read block 450 len 12288 05/30 Write block 214 len 20480 data: 8 7 6 8 8 06/30 Read block 303 len 40960 07/30 Write block 323 len 12288 data: 5 5 5 08/30 Read block 535 len 24576 09/30 Read block 403 len 36864 10/30 Write block 551 len 36864 data: 6 7 7 7 7 5 5 6 6 11/30 Flush 12/30 Write block 76 len 36864 data: 6 6 6 6 6 7 4 4 5 13/30 Read block 520 len 12288 14/30 Read block 105 len 28672 15/30 Read block 110 len 4096 16/30 Write block 328 len 8192 data: 4 4 17/30 Write block 29 len 12288 data: 2 2 2 18/30 Read block 26 len 4096 19/30 Write block 148 len 40960 data: 6 6 6 6 5 5 6 7 9 9 20/30 Read block 164 len 4096 21/30 Read block 87 len 20480 22/30 Flush 23/30 Write block 495 len 28672 data: 7 6 6 5 6 5 5 24/30 Write block 9 len 24576 data: 3 5 5 5 5 6 25/30 Read block 126 len 40960 26/30 Flush 27/30 Write block 169 len 4096 data: 6 28/30 Write block 264 len 4096 data: 6 29/30 Write block 25 len 20480 data: 3 3 3 2 3 30/30 Read block 420 len 16384 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:17.784711534Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 83 83 83 F F F 1 020-039 48 48 48 82 82 82 T T T 2 040-059 46 46 46 79 79 79 F F F 3 060-079 48 48 48 82 82 82 F F F 4 080-099 48 48 48 82 82 82 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 42 42 42 74 74 74 T T T 9 180-199 46 46 46 79 79 79 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 46 46 46 79 79 79 F F F 12 240-259 46 46 46 79 79 79 F F F 13 260-279 48 48 48 81 81 81 T T T 14 280-299 46 46 46 79 79 79 F F F 15 300-319 42 42 42 74 74 74 F F F 16 320-339 48 48 48 82 82 82 F F F 17 340-359 46 46 46 79 79 79 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 46 46 46 79 79 79 F F F 22 440-459 44 44 44 77 77 77 F F F 23 460-479 44 44 44 77 77 77 F F F 24 480-499 48 48 48 83 83 83 F F F 25 500-519 48 48 48 83 83 83 F F F 26 520-539 45 45 45 78 78 78 F F F 27 540-559 48 48 48 81 81 81 F F F 28 560-579 45 45 45 78 78 78 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 48, Max flush: 83 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 83 83 83 F F F 1 020-039 49 49 49 84 84 84 F F F 2 040-059 46 46 46 79 79 79 F F F 3 060-079 48 48 48 82 82 82 F F F 4 080-099 48 48 48 82 82 82 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 49 49 49 84 84 84 F F F 9 180-199 46 46 46 79 79 79 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 46 46 46 79 79 79 F F F 12 240-259 46 46 46 79 79 79 F F F 13 260-279 49 49 49 84 84 84 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 42 42 42 74 74 74 F F F 16 320-339 48 48 48 82 82 82 F F F 17 340-359 46 46 46 79 79 79 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 46 46 46 79 79 79 F F F 22 440-459 44 44 44 77 77 77 F F F 23 460-479 44 44 44 77 77 77 F F F 24 480-499 48 48 48 83 83 83 F F F 25 500-519 48 48 48 83 83 83 F F F 26 520-539 45 45 45 78 78 78 F F F 27 540-559 48 48 48 81 81 81 F F F 28 560-579 45 45 45 78 78 78 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 49, Max flush: 84 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.018614567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.064015822Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.064846128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.064883023Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.064898751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.064914039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} {"msg":"Crucible a8128a34-cc25-4c30-94f2-cfa90505429c has session id: d7683ccf-194d-4ee6-a10e-3abb84d7c214","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.06541286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Upstairs opts: Upstairs UUID: a8128a34-cc25-4c30-94f2-cfa90505429c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.065471532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Crucible stats registered with UUID: a8128a34-cc25-4c30-94f2-cfa90505429c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.065510637Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:22.065534563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066030455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066055662Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} {"msg":"a8128a34-cc25-4c30-94f2-cfa90505429c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066127012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066153338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"0","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066183274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"0","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066198252Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"1","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066219499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"1","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066245925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"2","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066273672Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"2","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066298778Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066327564Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066354141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"ds_connection connected from Ok(127.0.0.1:62517)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066381227Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"ds_connection connected from Ok(127.0.0.1:48362)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066411913Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"ds_connection connected from Ok(127.0.0.1:40357)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.066441809Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.067142002Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"2","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.067181887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"0","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.067832917Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"0","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.067983537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"2","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.068214575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"1","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.068441814Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.068621759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.068776968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"1","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[0]R flush_numbers[0..12]: [83, 84, 79, 82, 82, 77, 77, 82, 84, 79, 81, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069309294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[0]R generation[0..12]: [48, 49, 46, 48, 48, 44, 44, 48, 49, 46, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069332421Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069348559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[1]R flush_numbers[0..12]: [83, 84, 79, 82, 82, 77, 77, 82, 84, 79, 81, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069363027Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[1]R generation[0..12]: [48, 49, 46, 48, 48, 44, 44, 48, 49, 46, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069380465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069395632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[2]R flush_numbers[0..12]: [83, 84, 79, 82, 82, 77, 77, 82, 84, 79, 81, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.06940991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[2]R generation[0..12]: [48, 49, 46, 48, 48, 44, 44, 48, 49, 46, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069424208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069438537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069458474Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Generation requested: 50 >= found:50","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069491709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069508417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069522155Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"":"downstairs","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069535743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069547971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"0","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.0695617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"1","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069575308Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"client":"2","":"downstairs","session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069588926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069601574Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"a8128a34-cc25-4c30-94f2-cfa90505429c is now active with session: d7683ccf-194d-4ee6-a10e-3abb84d7c214","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069617792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.06963033Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392,"session_id":"d7683ccf-194d-4ee6-a10e-3abb84d7c214"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069642928Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} {"msg":"Activated sub_volume a8128a34-cc25-4c30-94f2-cfa90505429c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:22.069654437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 312 len 28672 data: 4 4 4 4 4 3 4 02/30 Read block 314 len 8192 03/30 Write block 332 len 20480 data: 3 4 5 5 7 04/30 Write block 425 len 28672 data: 5 4 5 4 4 4 3 05/30 Read block 415 len 28672 06/30 Write block 511 len 28672 data: 5 6 7 9 9 6 4 07/30 Read block 526 len 24576 08/30 Read block 364 len 12288 09/30 Flush 10/30 Write block 503 len 16384 data: 5 5 4 5 11/30 Write block 343 len 24576 data: 6 7 6 5 4 4 12/30 Read block 505 len 8192 13/30 Write block 239 len 28672 data: 7 8 7 5 7 7 7 14/30 Flush 15/30 Write block 578 len 8192 data: 5 6 16/30 Flush 17/30 Write block 187 len 28672 data: 3 4 4 5 5 4 4 18/30 Read block 592 len 12288 19/30 Read block 551 len 24576 20/30 Write block 443 len 28672 data: 5 4 6 6 9 9 9 21/30 Read block 307 len 36864 22/30 Read block 148 len 8192 23/30 Write block 514 len 40960 data: 10 10 7 5 4 4 6 6 6 5 24/30 Read block 200 len 4096 25/30 Read block 219 len 24576 26/30 Write block 556 len 36864 data: 6 6 7 7 7 9 9 8 9 27/30 Write block 12 len 16384 data: 6 6 7 5 28/30 Read block 373 len 8192 29/30 Read block 327 len 12288 30/30 Write block 534 len 20480 data: 5 5 6 6 5 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:22.091267914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 48 48 48 83 83 83 T T T 1 020-039 49 49 49 84 84 84 F F F 2 040-059 46 46 46 79 79 79 F F F 3 060-079 48 48 48 82 82 82 F F F 4 080-099 48 48 48 82 82 82 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 49 49 49 84 84 84 F F F 9 180-199 46 46 46 79 79 79 T T T 10 200-219 48 48 48 81 81 81 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 49 49 49 84 84 84 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 50 50 50 85 85 85 F F F 17 340-359 50 50 50 86 86 86 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 50 50 50 85 85 85 F F F 22 440-459 44 44 44 77 77 77 T T T 23 460-479 44 44 44 77 77 77 F F F 24 480-499 48 48 48 83 83 83 F F F 25 500-519 50 50 50 86 86 86 T T T 26 520-539 45 45 45 78 78 78 T T T 27 540-559 48 48 48 81 81 81 T T T 28 560-579 50 50 50 87 87 87 T T T 29 580-599 40 40 40 68 68 68 F F F Max gen: 50, Max flush: 87 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 51 51 51 88 88 88 F F F 1 020-039 49 49 49 84 84 84 F F F 2 040-059 46 46 46 79 79 79 F F F 3 060-079 48 48 48 82 82 82 F F F 4 080-099 48 48 48 82 82 82 F F F 5 100-119 44 44 44 77 77 77 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 49 49 49 84 84 84 F F F 9 180-199 51 51 51 88 88 88 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 50 50 50 86 86 86 F F F 13 260-279 49 49 49 84 84 84 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 50 50 50 85 85 85 F F F 17 340-359 50 50 50 86 86 86 F F F 18 360-379 48 48 48 80 80 80 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 50 50 50 85 85 85 F F F 22 440-459 51 51 51 88 88 88 F F F 23 460-479 44 44 44 77 77 77 F F F 24 480-499 48 48 48 83 83 83 F F F 25 500-519 51 51 51 88 88 88 F F F 26 520-539 51 51 51 88 88 88 F F F 27 540-559 51 51 51 88 88 88 F F F 28 560-579 51 51 51 88 88 88 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 51, Max flush: 88 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.346982027Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:26.390387567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.390834686Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.422809598Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.423769596Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.42380858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.423823038Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.423837626Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} {"msg":"Crucible 04fd9410-025b-4f6f-bec7-ff966e50e4dd has session id: fd7d3b3d-8096-4393-94a9-c0e6fce76288","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.424475839Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Upstairs opts: Upstairs UUID: 04fd9410-025b-4f6f-bec7-ff966e50e4dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.424523482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Crucible stats registered with UUID: 04fd9410-025b-4f6f-bec7-ff966e50e4dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.424546629Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:26.424561007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.424929236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.424956222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} {"msg":"04fd9410-025b-4f6f-bec7-ff966e50e4dd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425040101Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425059958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"0","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425089434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"0","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425110071Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"1","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425125039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"1","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425138797Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"2","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425153595Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"2","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425167344Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425183121Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425203048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"ds_connection connected from Ok(127.0.0.1:42296)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425308724Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"ds_connection connected from Ok(127.0.0.1:56807)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425349718Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"ds_connection connected from Ok(127.0.0.1:39681)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.425378195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.426054162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"1","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.426082758Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"2","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.426226238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"0","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.426686824Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"1","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.42671912Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"2","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.42693586Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"0","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427267215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427346604Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[0]R flush_numbers[0..12]: [88, 84, 79, 82, 82, 77, 77, 82, 84, 88, 81, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427459508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[0]R generation[0..12]: [51, 49, 46, 48, 48, 44, 44, 48, 49, 51, 48, 50]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427482005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427497863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[1]R flush_numbers[0..12]: [88, 84, 79, 82, 82, 77, 77, 82, 84, 88, 81, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427512791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[1]R generation[0..12]: [51, 49, 46, 48, 48, 44, 44, 48, 49, 51, 48, 50]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427527559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427542237Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[2]R flush_numbers[0..12]: [88, 84, 79, 82, 82, 77, 77, 82, 84, 88, 81, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427556485Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[2]R generation[0..12]: [51, 49, 46, 48, 48, 44, 44, 48, 49, 51, 48, 50]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427570533Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427585011Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427598979Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427617316Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-18T05:04:26.427637684Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427651562Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"":"downstairs","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.4276654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427677848Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"0","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427691726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"1","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427705374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"client":"2","":"downstairs","session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427719082Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427731861Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"04fd9410-025b-4f6f-bec7-ff966e50e4dd is now active with session: fd7d3b3d-8096-4393-94a9-c0e6fce76288","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427744939Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427757147Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400,"session_id":"fd7d3b3d-8096-4393-94a9-c0e6fce76288"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427769666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} {"msg":"Activated sub_volume 04fd9410-025b-4f6f-bec7-ff966e50e4dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.427781054Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 4096 data: 8 02/30 Read block 78 len 20480 03/30 Write block 194 len 12288 data: 4 3 3 04/30 Read block 129 len 4096 05/30 Write block 13 len 36864 data: 7 8 6 6 6 6 6 4 3 06/30 Write block 533 len 32768 data: 5 6 6 7 7 6 7 5 07/30 Write block 356 len 16384 data: 6 5 5 4 08/30 Write block 483 len 20480 data: 6 7 7 5 4 09/30 Flush 10/30 Write block 206 len 32768 data: 5 5 5 6 6 4 5 8 11/30 Write block 538 len 24576 data: 7 8 6 4 4 4 12/30 Read block 586 len 40960 13/30 Read block 552 len 24576 14/30 Write block 370 len 16384 data: 4 4 4 4 15/30 Write block 160 len 12288 data: 6 5 7 16/30 Read block 126 len 8192 17/30 Read block 494 len 40960 18/30 Read block 62 len 8192 19/30 Read block 238 len 40960 20/30 Write block 464 len 16384 data: 3 3 4 4 21/30 Write block 99 len 12288 data: 9 8 7 22/30 Read block 361 len 24576 23/30 Flush 24/30 Write block 265 len 8192 data: 5 4 25/30 Read block 34 len 12288 26/30 Read block 184 len 12288 27/30 Write block 175 len 20480 data: 3 2 2 3 3 28/30 Read block 443 len 16384 29/30 Write block 468 len 16384 data: 6 5 5 6 30/30 Write block 255 len 24576 data: 6 7 7 7 8 8 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:26.447261554Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 89 89 89 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 46 46 46 79 79 79 F F F 3 060-079 48 48 48 82 82 82 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 52 52 52 90 90 90 T T T 9 180-199 52 52 52 89 89 89 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 50 50 50 86 86 86 T T T 13 260-279 49 49 49 84 84 84 T T T 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 52 52 52 89 89 89 F F F 17 340-359 52 52 52 89 89 89 F F F 18 360-379 52 52 52 90 90 90 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 50 50 50 85 85 85 F F F 22 440-459 51 51 51 88 88 88 F F F 23 460-479 52 52 52 90 90 90 T T T 24 480-499 52 52 52 89 89 89 F F F 25 500-519 51 51 51 88 88 88 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 52 52 52 90 90 90 F F F 28 560-579 51 51 51 88 88 88 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 52, Max flush: 90 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 89 89 89 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 46 46 46 79 79 79 F F F 3 060-079 48 48 48 82 82 82 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 44 44 44 77 77 77 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 53 53 53 91 91 91 F F F 9 180-199 52 52 52 89 89 89 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 53 53 53 91 91 91 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 52 52 52 89 89 89 F F F 17 340-359 52 52 52 89 89 89 F F F 18 360-379 52 52 52 90 90 90 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 50 50 50 85 85 85 F F F 22 440-459 51 51 51 88 88 88 F F F 23 460-479 53 53 53 91 91 91 F F F 24 480-499 52 52 52 89 89 89 F F F 25 500-519 51 51 51 88 88 88 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 52 52 52 90 90 90 F F F 28 560-579 51 51 51 88 88 88 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 53, Max flush: 91 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.714312319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:30.757871398Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.758300399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.790370448Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.791017899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.791062353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.79107995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.791098238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} {"msg":"Crucible ec8ae296-6e7d-4e67-a5ed-d04a6f641b44 has session id: ae392786-6644-4cfd-a34e-6a950b13869b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.791628675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Upstairs opts: Upstairs UUID: ec8ae296-6e7d-4e67-a5ed-d04a6f641b44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.79166501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Crucible stats registered with UUID: ec8ae296-6e7d-4e67-a5ed-d04a6f641b44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.791692516Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:30.791710733Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792091821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792116408Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} {"msg":"ec8ae296-6e7d-4e67-a5ed-d04a6f641b44 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792198086Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792216264Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"0","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.79224551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"0","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792261468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"1","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792277266Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"1","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792291054Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"2","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792305522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"2","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.79231918Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792333958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792348436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60220)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792407188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35991)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792446682Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49050)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.792474668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.793303145Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"2","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.793347728Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"1","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.793440546Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"0","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794127941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"1","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794171085Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"2","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794204161Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"0","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794688564Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.79479034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[0]R flush_numbers[0..12]: [89, 89, 79, 82, 90, 90, 77, 82, 91, 89, 90, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794857941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[0]R generation[0..12]: [52, 52, 46, 48, 52, 52, 44, 48, 53, 52, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794878578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794893996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[1]R flush_numbers[0..12]: [89, 89, 79, 82, 90, 90, 77, 82, 91, 89, 90, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794909624Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[1]R generation[0..12]: [52, 52, 46, 48, 52, 52, 44, 48, 53, 52, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794927461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794942639Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[2]R flush_numbers[0..12]: [89, 89, 79, 82, 90, 90, 77, 82, 91, 89, 90, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794956977Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[2]R generation[0..12]: [52, 52, 46, 48, 52, 52, 44, 48, 53, 52, 52, 50]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794970995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.794985323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795004671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795025988Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795052024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795073511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"":"downstairs","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795095708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":Wait for a query_work_queue command to finish before sending IO 1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795125194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"0","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.79515238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"1","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795171188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"client":"2","":"downstairs","session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795185276Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795197954Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"ec8ae296-6e7d-4e67-a5ed-d04a6f641b44 is now active with session: ae392786-6644-4cfd-a34e-6a950b13869b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.79522211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795246167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408,"session_id":"ae392786-6644-4cfd-a34e-6a950b13869b"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795268674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} {"msg":"Activated sub_volume ec8ae296-6e7d-4e67-a5ed-d04a6f641b44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.795281722Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 220 len 36864 data: 8 8 7 7 7 6 7 6 6 02/30 Write block 126 len 16384 data: 6 5 5 5 03/30 Write block 425 len 8192 data: 6 5 04/30 Flush 05/30 Write block 185 len 36864 data: 3 3 4 5 5 6 6 5 5 06/30 Write block 107 len 4096 data: 5 07/30 Write block 443 len 32768 data: 6 5 7 7 10 10 10 9 08/30 Flush 09/30 Write block 246 len 12288 data: 7 6 7 10/30 Write block 366 len 28672 data: 3 3 3 4 5 5 5 11/30 Read block 543 len 24576 12/30 Write block 109 len 36864 data: 5 5 4 2 2 2 3 4 4 13/30 Read block 214 len 28672 14/30 Write block 325 len 20480 data: 6 4 4 5 5 15/30 Write block 92 len 24576 data: 9 9 8 7 7 7 16/30 Read block 16 len 32768 17/30 Read block 155 len 40960 18/30 Read block 525 len 4096 19/30 Read block 274 len 36864 20/30 Flush 21/30 Flush 22/30 Write block 571 len 20480 data: 4 3 2 2 2 23/30 Read block 465 len 12288 24/30 Write block 471 len 24576 data: 7 7 8 6 6 6 25/30 Read block 37 len 40960 26/30 Read block 138 len 20480 27/30 Write block 545 len 16384 data: 5 5 5 5 28/30 Read block 193 len 36864 29/30 Write block 18 len 8192 data: 7 7 30/30 Write block 47 len 40960 data: 8 5 3 3 4 4 5 5 7 6 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:30.81623476Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 89 89 89 T T T 1 020-039 52 52 52 89 89 89 F F F 2 040-059 46 46 46 79 79 79 T T T 3 060-079 48 48 48 82 82 82 F F F 4 080-099 54 54 54 94 94 94 F F F 5 100-119 54 54 54 94 94 94 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 53 53 53 91 91 91 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 54 54 54 92 92 92 F F F 12 240-259 54 54 54 94 94 94 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 54 54 54 94 94 94 F F F 17 340-359 52 52 52 89 89 89 F F F 18 360-379 54 54 54 94 94 94 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 54 54 54 93 93 93 F F F 23 460-479 53 53 53 91 91 91 T T T 24 480-499 52 52 52 89 89 89 F F F 25 500-519 51 51 51 88 88 88 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 52 52 52 90 90 90 T T T 28 560-579 51 51 51 88 88 88 T T T 29 580-599 40 40 40 68 68 68 F F F Max gen: 54, Max flush: 94 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 95 95 95 F F F 1 020-039 52 52 52 89 89 89 F F F 2 040-059 55 55 55 95 95 95 F F F 3 060-079 48 48 48 82 82 82 F F F 4 080-099 54 54 54 94 94 94 F F F 5 100-119 54 54 54 94 94 94 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 48 48 48 82 82 82 F F F 8 160-179 53 53 53 91 91 91 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 54 54 54 92 92 92 F F F 12 240-259 54 54 54 94 94 94 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 54 54 54 94 94 94 F F F 17 340-359 52 52 52 89 89 89 F F F 18 360-379 54 54 54 94 94 94 F F F 19 380-399 44 44 44 77 77 77 F F F 20 400-419 46 46 46 79 79 79 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 54 54 54 93 93 93 F F F 23 460-479 55 55 55 95 95 95 F F F 24 480-499 52 52 52 89 89 89 F F F 25 500-519 51 51 51 88 88 88 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 55, Max flush: 95 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.092266829Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.139764017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.140485808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.140525982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.14054019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.140555098Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} {"msg":"Crucible c9345045-c598-4120-9ddd-7f821bcb852c has session id: 4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141085935Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Upstairs opts: Upstairs UUID: c9345045-c598-4120-9ddd-7f821bcb852c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.14119149Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Crucible stats registered with UUID: c9345045-c598-4120-9ddd-7f821bcb852c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141221656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:35.141245363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141720078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141746014Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} {"msg":"c9345045-c598-4120-9ddd-7f821bcb852c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141831642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.14184937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"0","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141878006Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"0","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141893134Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"1","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141908122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"1","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141934648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"2","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141963544Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"2","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.141984791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.142000839Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.142015457Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52145)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.142189733Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49948)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.142236157Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55638)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.142266883Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.142977845Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"2","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.143006521Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"0","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.143715823Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"2","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.143744609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"0","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.144336578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.144360884Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.144374873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"1","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.144954563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"1","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[0]R flush_numbers[0..12]: [95, 89, 95, 82, 94, 94, 92, 82, 91, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145471002Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[0]R generation[0..12]: [55, 52, 55, 48, 54, 54, 54, 48, 53, 54, 52, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145495199Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145510806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[1]R flush_numbers[0..12]: [95, 89, 95, 82, 94, 94, 92, 82, 91, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145525394Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[1]R generation[0..12]: [55, 52, 55, 48, 54, 54, 54, 48, 53, 54, 52, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145539652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.14555565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[2]R flush_numbers[0..12]: [95, 89, 95, 82, 94, 94, 92, 82, 91, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145570328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[2]R generation[0..12]: [55, 52, 55, 48, 54, 54, 54, 48, 53, 54, 52, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145584376Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145598804Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145612352Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145626281Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:35.145640079Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145660586Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"":"downstairs","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145674964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145687282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"0","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.14570116Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"1","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145714828Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"client":"2","":"downstairs","session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145728376Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145740915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"c9345045-c598-4120-9ddd-7f821bcb852c is now active with session: 4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145754123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145766441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416,"session_id":"4d6fb850-ccc5-4c49-acf4-b2ab7619ee6f"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145778869Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} {"msg":"Activated sub_volume c9345045-c598-4120-9ddd-7f821bcb852c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:35.145790458Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 477 len 16384 data: 6 4 5 5 02/30 Write block 160 len 12288 data: 7 6 8 03/30 Write block 406 len 12288 data: 4 5 4 04/30 Read block 304 len 16384 05/30 Flush 06/30 Write block 67 len 4096 data: 8 07/30 Read block 374 len 12288 08/30 Flush 09/30 Write block 170 len 32768 data: 6 6 5 5 4 4 3 3 10/30 Write block 228 len 24576 data: 7 4 5 4 4 4 11/30 Write block 493 len 32768 data: 6 7 8 7 7 6 7 6 12/30 Read block 334 len 32768 13/30 Read block 255 len 12288 14/30 Read block 421 len 24576 15/30 Read block 310 len 24576 16/30 Write block 143 len 36864 data: 3 4 6 6 6 7 7 7 7 17/30 Write block 109 len 20480 data: 6 6 5 3 3 18/30 Read block 357 len 24576 19/30 Flush 20/30 Write block 386 len 12288 data: 4 4 5 21/30 Write block 107 len 24576 data: 6 6 7 7 6 4 22/30 Write block 400 len 20480 data: 5 5 5 6 5 23/30 Write block 355 len 24576 data: 5 7 6 6 5 4 24/30 Read block 321 len 4096 25/30 Write block 17 len 32768 data: 7 8 8 5 4 3 3 3 26/30 Read block 267 len 28672 27/30 Read block 101 len 8192 28/30 Read block 182 len 28672 29/30 Write block 61 len 12288 data: 8 9 9 30/30 Read block 29 len 40960 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 3 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:35.166051611Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 95 95 95 T T T 1 020-039 52 52 52 89 89 89 T T T 2 040-059 55 55 55 95 95 95 F F F 3 060-079 56 56 56 97 97 97 T T T 4 080-099 54 54 54 94 94 94 F F F 5 100-119 56 56 56 98 98 98 T T T 6 120-139 54 54 54 92 92 92 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 56 56 56 98 98 98 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 56 56 56 98 98 98 F F F 12 240-259 54 54 54 94 94 94 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 54 54 54 94 94 94 F F F 17 340-359 52 52 52 89 89 89 T T T 18 360-379 54 54 54 94 94 94 T T T 19 380-399 44 44 44 77 77 77 T T T 20 400-419 56 56 56 96 96 96 T T T 21 420-439 54 54 54 92 92 92 F F F 22 440-459 54 54 54 93 93 93 F F F 23 460-479 56 56 56 96 96 96 F F F 24 480-499 56 56 56 98 98 98 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 56, Max flush: 98 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 57 57 57 99 99 99 F F F 2 040-059 55 55 55 95 95 95 F F F 3 060-079 57 57 57 99 99 99 F F F 4 080-099 54 54 54 94 94 94 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 56 56 56 98 98 98 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 56 56 56 98 98 98 F F F 12 240-259 54 54 54 94 94 94 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 54 54 54 94 94 94 F F F 17 340-359 57 57 57 99 99 99 F F F 18 360-379 57 57 57 99 99 99 F F F 19 380-399 57 57 57 99 99 99 F F F 20 400-419 57 57 57 99 99 99 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 54 54 54 93 93 93 F F F 23 460-479 56 56 56 96 96 96 F F F 24 480-499 56 56 56 98 98 98 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 40 40 40 68 68 68 F F F Max gen: 57, Max flush: 99 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.426017588Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:39.468880083Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.46926368Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.500620147Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.501295044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.501337918Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.501355256Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.501373394Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} {"msg":"Crucible dbd839e9-c3ae-4368-bede-437015e7b98f has session id: 8b52427a-ec1b-49b7-86e1-72e6c873c1ce","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.501925168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Upstairs opts: Upstairs UUID: dbd839e9-c3ae-4368-bede-437015e7b98f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.50197978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Crucible stats registered with UUID: dbd839e9-c3ae-4368-bede-437015e7b98f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502019465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:39.502037912Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502477342Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502521706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} {"msg":"dbd839e9-c3ae-4368-bede-437015e7b98f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502586877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502615423Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"0","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502649248Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"0","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502665996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"1","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502680914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"1","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502696672Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"2","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502711989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"2","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502732887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502760743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502788439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:62312)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502931319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:42007)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.502974633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:54444)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.503127872Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.503696304Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"2","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.503750427Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"1","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.503788761Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"0","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.504286163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"2","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.504433193Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"1","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.50459723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"0","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.504955651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505050308Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[0]R flush_numbers[0..12]: [99, 99, 95, 99, 94, 99, 92, 98, 98, 93, 90, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505160163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[0]R generation[0..12]: [57, 57, 55, 57, 54, 57, 54, 56, 56, 54, 52, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.50517975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505194788Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[1]R flush_numbers[0..12]: [99, 99, 95, 99, 94, 99, 92, 98, 98, 93, 90, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505209026Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[1]R generation[0..12]: [57, 57, 55, 57, 54, 57, 54, 56, 56, 54, 52, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505222854Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505240412Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[2]R flush_numbers[0..12]: [99, 99, 95, 99, 94, 99, 92, 98, 98, 93, 90, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.50525497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[2]R generation[0..12]: [57, 57, 55, 57, 54, 57, 54, 56, 56, 54, 52, 56]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505269328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505283616Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505297064Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505310602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.50532395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505344518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"":"downstairs","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505358296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505370554Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"0","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505384252Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"1","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.50539773Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"client":"2","":"downstairs","session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505411438Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505423996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"dbd839e9-c3ae-4368-bede-437015e7b98f is now active with session: 8b52427a-ec1b-49b7-86e1-72e6c873c1ce","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505437185Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505451673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424,"session_id":"8b52427a-ec1b-49b7-86e1-72e6c873c1ce"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505464281Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} {"msg":"Activated sub_volume dbd839e9-c3ae-4368-bede-437015e7b98f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:39.505475769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 380 len 16384 data: 3 3 3 3 02/30 Flush 03/30 Write block 76 len 12288 data: 7 7 7 04/30 Write block 595 len 16384 data: 4 4 4 3 05/30 Read block 292 len 36864 06/30 Read block 247 len 28672 07/30 Read block 532 len 20480 08/30 Read block 191 len 20480 09/30 Write block 120 len 28672 data: 6 8 7 5 5 6 7 10/30 Read block 10 len 12288 11/30 Write block 24 len 4096 data: 4 12/30 Read block 493 len 16384 13/30 Read block 413 len 24576 14/30 Read block 355 len 16384 15/30 Read block 530 len 28672 16/30 Write block 61 len 4096 data: 9 17/30 Write block 224 len 28672 data: 8 7 8 7 8 5 6 18/30 Write block 364 len 28672 data: 4 4 4 4 4 5 6 19/30 Read block 170 len 28672 20/30 Read block 410 len 40960 21/30 Read block 49 len 28672 22/30 Read block 20 len 4096 23/30 Write block 263 len 40960 data: 7 7 6 5 4 4 2 2 3 2 24/30 Read block 431 len 8192 25/30 Write block 253 len 28672 data: 3 5 7 8 8 8 9 26/30 Write block 57 len 12288 data: 6 6 7 27/30 Write block 365 len 24576 data: 5 5 5 5 6 7 28/30 Read block 257 len 32768 29/30 Write block 232 len 20480 data: 5 5 3 5 6 30/30 Read block 221 len 4096 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 28 26 26 80 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:39.526232154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 57 57 57 99 99 99 T T T 2 040-059 55 55 55 95 95 95 T T T 3 060-079 57 57 57 99 99 99 T T T 4 080-099 54 54 54 94 94 94 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 54 54 54 92 92 92 T T T 7 140-159 56 56 56 98 98 98 F F F 8 160-179 56 56 56 98 98 98 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 56 56 56 98 98 98 T T T 12 240-259 54 54 54 94 94 94 T T T 13 260-279 53 53 53 91 91 91 T T T 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 54 54 54 94 94 94 F F F 17 340-359 57 57 57 99 99 99 F F F 18 360-379 57 57 57 99 99 99 T T T 19 380-399 58 58 58 100 100 100 F F F 20 400-419 57 57 57 99 99 99 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 54 54 54 93 93 93 F F F 23 460-479 56 56 56 96 96 96 F F F 24 480-499 56 56 56 98 98 98 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 40 40 40 68 68 68 T T T Max gen: 58, Max flush: 100 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 99 99 99 F F F 1 020-039 59 59 59 101 101 101 F F F 2 040-059 59 59 59 101 101 101 F F F 3 060-079 59 59 59 101 101 101 F F F 4 080-099 54 54 54 94 94 94 F F F 5 100-119 57 57 57 99 99 99 F F F 6 120-139 59 59 59 101 101 101 F F F 7 140-159 56 56 56 98 98 98 F F F 8 160-179 56 56 56 98 98 98 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 59 59 59 101 101 101 F F F 12 240-259 59 59 59 101 101 101 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 46 46 46 79 79 79 F F F 15 300-319 50 50 50 85 85 85 F F F 16 320-339 54 54 54 94 94 94 F F F 17 340-359 57 57 57 99 99 99 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 58 58 58 100 100 100 F F F 20 400-419 57 57 57 99 99 99 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 54 54 54 93 93 93 F F F 23 460-479 56 56 56 96 96 96 F F F 24 480-499 56 56 56 98 98 98 F F F 25 500-519 56 56 56 98 98 98 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 59, Max flush: 101 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.818496419Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.86320609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.864039745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.864082679Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.864100297Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.864120194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} {"msg":"Crucible 223f3041-973d-4422-996a-79c89abe0f9f has session id: 057883b9-73e0-42b3-8a40-b2a0a4dfdc20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.864706173Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Upstairs opts: Upstairs UUID: 223f3041-973d-4422-996a-79c89abe0f9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.864751277Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Crucible stats registered with UUID: 223f3041-973d-4422-996a-79c89abe0f9f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.864773944Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:43.864793652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865256148Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865284194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} {"msg":"223f3041-973d-4422-996a-79c89abe0f9f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865346605Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865365523Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"0","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865395419Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"0","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865414356Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"1","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865430504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"1","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865449091Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"2","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865475858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"2","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865502804Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.86553046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865558956Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"ds_connection connected from Ok(127.0.0.1:46858)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865585643Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"ds_connection connected from Ok(127.0.0.1:46696)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865630916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"ds_connection connected from Ok(127.0.0.1:51796)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.865659442Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.866338479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"1","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.866380023Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"0","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.867013816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"1","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.867044792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"0","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.867670916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.867707041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.868349972Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"2","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.868833736Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"2","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[0]R flush_numbers[0..12]: [99, 101, 101, 101, 94, 99, 101, 98, 98, 93, 90, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869342356Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[0]R generation[0..12]: [57, 59, 59, 59, 54, 57, 59, 56, 56, 54, 52, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869368142Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.8693834Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[1]R flush_numbers[0..12]: [99, 101, 101, 101, 94, 99, 101, 98, 98, 93, 90, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869398428Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[1]R generation[0..12]: [57, 59, 59, 59, 54, 57, 59, 56, 56, 54, 52, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869412916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869427524Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[2]R flush_numbers[0..12]: [99, 101, 101, 101, 94, 99, 101, 98, 98, 93, 90, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869441682Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[2]R generation[0..12]: [57, 59, 59, 59, 54, 57, 59, 56, 56, 54, 52, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.86945568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869470088Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869483496Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869500594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869521351Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869535349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"":"downstairs","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869549177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869561516Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"0","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869575574Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"1","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869589362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"client":"2","":"downstairs","session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.86960312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869615968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"223f3041-973d-4422-996a-79c89abe0f9f is now active with session: 057883b9-73e0-42b3-8a40-b2a0a4dfdc20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869629196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869641595Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432,"session_id":"057883b9-73e0-42b3-8a40-b2a0a4dfdc20"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:43.869654163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} {"msg":"Activated sub_volume 223f3041-973d-4422-996a-79c89abe0f9f","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-18T05:04:43.86967924Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 395 len 40960 data: 3 5 4 4 4 6 6 6 7 6 02/30 Write block 496 len 36864 data: 8 8 7 8 7 6 5 6 6 03/30 Read block 140 len 16384 04/30 Write block 273 len 32768 data: 4 3 4 3 3 5 5 7 05/30 Read block 578 len 24576 06/30 Write block 85 len 8192 data: 5 6 07/30 Write block 119 len 24576 data: 6 7 9 8 6 6 08/30 Read block 474 len 8192 09/30 Write block 230 len 32768 data: 7 5 6 6 4 6 7 8 10/30 Flush 11/30 Write block 319 len 12288 data: 4 3 4 12/30 Write block 163 len 8192 data: 7 7 13/30 Read block 440 len 12288 14/30 Read block 554 len 4096 15/30 Read block 157 len 28672 16/30 Write block 5 len 12288 data: 3 4 3 17/30 Read block 189 len 8192 18/30 Read block 306 len 24576 19/30 Read block 153 len 24576 20/30 Write block 140 len 32768 data: 4 4 4 4 5 7 7 7 21/30 Write block 392 len 24576 data: 2 2 3 4 6 5 22/30 Write block 448 len 24576 data: 11 11 10 9 7 6 23/30 Write block 222 len 16384 data: 8 8 9 8 24/30 Write block 392 len 28672 data: 3 3 4 5 7 6 5 25/30 Read block 463 len 20480 26/30 Flush 27/30 Write block 457 len 20480 data: 4 5 5 3 3 28/30 Read block 117 len 36864 29/30 Read block 217 len 24576 30/30 Read block 529 len 12288 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:43.89090716Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 59 59 59 101 101 101 F F F 2 040-059 59 59 59 101 101 101 F F F 3 060-079 59 59 59 101 101 101 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 60 60 60 102 102 102 F F F 6 120-139 60 60 60 102 102 102 F F F 7 140-159 60 60 60 103 103 103 F F F 8 160-179 60 60 60 103 103 103 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 60 60 60 103 103 103 F F F 12 240-259 59 59 59 101 101 101 F F F 13 260-279 60 60 60 102 102 102 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 60 60 60 103 103 103 F F F 17 340-359 57 57 57 99 99 99 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 60 60 60 102 102 102 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 60 60 60 103 103 103 T T T 23 460-479 56 56 56 96 96 96 T T T 24 480-499 60 60 60 102 102 102 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 60, Max flush: 103 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 59 59 59 101 101 101 F F F 2 040-059 59 59 59 101 101 101 F F F 3 060-079 59 59 59 101 101 101 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 60 60 60 102 102 102 F F F 6 120-139 60 60 60 102 102 102 F F F 7 140-159 60 60 60 103 103 103 F F F 8 160-179 60 60 60 103 103 103 F F F 9 180-199 54 54 54 93 93 93 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 60 60 60 103 103 103 F F F 12 240-259 59 59 59 101 101 101 F F F 13 260-279 60 60 60 102 102 102 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 60 60 60 103 103 103 F F F 17 340-359 57 57 57 99 99 99 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 60 60 60 102 102 102 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 60 60 60 102 102 102 F F F 25 500-519 60 60 60 102 102 102 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 61, Max flush: 104 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.129999488Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:48.174643468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.175100675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.207130919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.208110785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.20814768Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.208162087Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.208177765Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} {"msg":"Crucible 385494b7-0c9f-4858-80ef-b4be85595920 has session id: 2e95b392-5858-4d62-8387-45f0fa2f94e8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.208816967Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Upstairs opts: Upstairs UUID: 385494b7-0c9f-4858-80ef-b4be85595920, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.20887182Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Crucible stats registered with UUID: 385494b7-0c9f-4858-80ef-b4be85595920","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.208892887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:48.208906375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209310529Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209339556Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} {"msg":"385494b7-0c9f-4858-80ef-b4be85595920 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209368252Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.20938196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"0","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209410606Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"0","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209424994Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"1","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209440082Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"1","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.20945372Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"2","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209468398Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"2","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209481996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209496914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209511482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:58856)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209654652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:57693)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209700746Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49388)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.209729102Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.210440404Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"1","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.210478409Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"2","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.210572126Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"0","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211170374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"1","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.21119969Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"2","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211223686Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"0","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211809856Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211838262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[0]R flush_numbers[0..12]: [103, 101, 101, 101, 102, 102, 102, 103, 103, 93, 90, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.2118532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[0]R generation[0..12]: [60, 59, 59, 59, 60, 60, 60, 60, 60, 54, 52, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211868928Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211884235Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[1]R flush_numbers[0..12]: [103, 101, 101, 101, 102, 102, 102, 103, 103, 93, 90, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211898824Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[1]R generation[0..12]: [60, 59, 59, 59, 60, 60, 60, 60, 60, 54, 52, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211916241Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211931099Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[2]R flush_numbers[0..12]: [103, 101, 101, 101, 102, 102, 102, 103, 103, 93, 90, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211945137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[2]R generation[0..12]: [60, 59, 59, 59, 60, 60, 60, 60, 60, 54, 52, 60]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211959205Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211973673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.211987101Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212023926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212038934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-03-18T05:04:48.212052472Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"":"downstairs","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212073089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212085688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"0","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212099436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"1","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212112754Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"client":"2","":"downstairs","session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212126222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.21213874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"385494b7-0c9f-4858-80ef-b4be85595920 is now active with session: 2e95b392-5858-4d62-8387-45f0fa2f94e8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212154078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212167547Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440,"session_id":"2e95b392-5858-4d62-8387-45f0fa2f94e8"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212180245Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} {"msg":"Activated sub_volume 385494b7-0c9f-4858-80ef-b4be85595920","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:48.212192083Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 192 len 4096 data: 6 02/30 Read block 315 len 36864 03/30 Read block 130 len 12288 04/30 Write block 71 len 4096 data: 5 05/30 Read block 12 len 4096 06/30 Flush 07/30 Write block 187 len 20480 data: 5 6 6 7 7 08/30 Write block 102 len 16384 data: 5 7 7 6 09/30 Write block 336 len 20480 data: 9 8 8 7 7 10/30 Write block 36 len 4096 data: 2 11/30 Read block 243 len 12288 12/30 Write block 38 len 36864 data: 2 6 8 10 11 11 10 9 8 13/30 Write block 232 len 24576 data: 7 7 5 7 8 9 14/30 Write block 505 len 20480 data: 5 6 4 4 3 15/30 Read block 440 len 20480 16/30 Read block 455 len 12288 17/30 Read block 478 len 28672 18/30 Write block 590 len 40960 data: 6 4 4 4 5 5 5 5 4 3 19/30 Flush 20/30 Write block 335 len 4096 data: 6 21/30 Write block 586 len 36864 data: 7 6 5 4 7 5 5 5 6 22/30 Write block 509 len 16384 data: 4 5 6 7 23/30 Write block 347 len 24576 data: 5 5 4 4 5 6 24/30 Write block 239 len 16384 data: 8 9 8 6 25/30 Flush 26/30 Write block 415 len 8192 data: 5 5 27/30 Read block 173 len 4096 28/30 Write block 548 len 28672 data: 6 6 6 7 8 8 8 29/30 Read block 22 len 40960 30/30 Read block 44 len 12288 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:48.231311574Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 62 62 62 106 106 106 F F F 3 060-079 62 62 62 105 105 105 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 62 62 62 106 106 106 F F F 6 120-139 60 60 60 102 102 102 F F F 7 140-159 60 60 60 103 103 103 F F F 8 160-179 60 60 60 103 103 103 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 62 62 62 107 107 107 F F F 13 260-279 60 60 60 102 102 102 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 60 60 60 102 102 102 T T T 21 420-439 54 54 54 92 92 92 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 60 60 60 102 102 102 F F F 25 500-519 62 62 62 107 107 107 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 55 55 55 95 95 95 T T T 28 560-579 55 55 55 95 95 95 F F F 29 580-599 62 62 62 107 107 107 F F F Max gen: 62, Max flush: 107 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 62 62 62 106 106 106 F F F 3 060-079 62 62 62 105 105 105 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 62 62 62 106 106 106 F F F 6 120-139 60 60 60 102 102 102 F F F 7 140-159 60 60 60 103 103 103 F F F 8 160-179 60 60 60 103 103 103 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 62 62 62 107 107 107 F F F 13 260-279 60 60 60 102 102 102 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 63 63 63 108 108 108 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 60 60 60 102 102 102 F F F 25 500-519 62 62 62 107 107 107 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 62 62 62 107 107 107 F F F Max gen: 63, Max flush: 108 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.494477912Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:52.538414639Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.538867947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.57076054Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.571509317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.571549792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.571566659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.571584337Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} {"msg":"Crucible b273b2c4-c442-4093-ac18-9a48e91ab3cb has session id: 1527ec8b-d099-49ac-a80d-604bbc401820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.572370009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Upstairs opts: Upstairs UUID: b273b2c4-c442-4093-ac18-9a48e91ab3cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.572413783Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Crucible stats registered with UUID: b273b2c4-c442-4093-ac18-9a48e91ab3cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.57243055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:52.572445528Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.572884668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.572909714Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} {"msg":"b273b2c4-c442-4093-ac18-9a48e91ab3cb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.572973556Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.572990893Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"0","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573020109Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"0","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573039087Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"1","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573054195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"1","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573067873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"2","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573082671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"2","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573100338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573125065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573151881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"ds_connection connected from Ok(127.0.0.1:61423)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573180767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"ds_connection connected from Ok(127.0.0.1:35466)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573213113Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"ds_connection connected from Ok(127.0.0.1:38310)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573372811Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.57396368Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"1","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.573992225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"2","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.574067785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"0","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.574665163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"1","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.574706217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"2","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.574769419Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"0","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575319093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575345179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[0]R flush_numbers[0..12]: [103, 106, 106, 105, 102, 106, 102, 103, 103, 106, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575373016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[0]R generation[0..12]: [60, 62, 62, 62, 60, 62, 60, 60, 60, 62, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575388713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575403451Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[1]R flush_numbers[0..12]: [103, 106, 106, 105, 102, 106, 102, 103, 103, 106, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.57541778Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[1]R generation[0..12]: [60, 62, 62, 62, 60, 62, 60, 60, 60, 62, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575432678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575447236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[2]R flush_numbers[0..12]: [103, 106, 106, 105, 102, 106, 102, 103, 103, 106, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575461343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[2]R generation[0..12]: [60, 62, 62, 62, 60, 62, 60, 60, 60, 62, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575475741Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.57549013Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575503458Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575516916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575534473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"All extents match",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.57555567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"":"downstairs","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575569469Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575581937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"0","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575595545Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"1","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575609053Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"client":"2","":"downstairs","session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575622781Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.57563513Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"b273b2c4-c442-4093-ac18-9a48e91ab3cb is now active with session: 1527ec8b-d099-49ac-a80d-604bbc401820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575648478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575660666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448,"session_id":"1527ec8b-d099-49ac-a80d-604bbc401820"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575673214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} {"msg":"Activated sub_volume b273b2c4-c442-4093-ac18-9a48e91ab3cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:52.575684633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 63 len 12288 data: 10 9 10 02/30 Read block 74 len 32768 03/30 Write block 182 len 40960 data: 3 3 3 4 4 6 7 7 8 8 04/30 Flush 05/30 Write block 143 len 24576 data: 5 6 8 8 8 8 06/30 Write block 108 len 8192 data: 7 8 07/30 Read block 92 len 16384 08/30 Write block 258 len 12288 data: 9 10 9 09/30 Read block 578 len 28672 10/30 Read block 152 len 24576 11/30 Write block 181 len 8192 data: 4 4 12/30 Write block 338 len 40960 data: 9 8 8 6 7 7 8 7 6 6 13/30 Read block 131 len 28672 14/30 Read block 65 len 16384 15/30 Read block 100 len 36864 16/30 Flush 17/30 Write block 59 len 28672 data: 8 8 10 10 11 10 11 18/30 Read block 366 len 8192 19/30 Read block 455 len 20480 20/30 Read block 313 len 8192 21/30 Write block 133 len 12288 data: 6 5 6 22/30 Flush 23/30 Write block 51 len 36864 data: 5 5 6 6 8 7 7 7 9 24/30 Write block 403 len 36864 data: 8 7 5 5 6 5 4 4 4 25/30 Read block 273 len 40960 26/30 Read block 369 len 24576 27/30 Read block 583 len 28672 28/30 Read block 212 len 32768 29/30 Write block 245 len 4096 data: 8 30/30 Read block 308 len 28672 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:52.596734808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 64 64 64 111 111 111 T T T 3 060-079 64 64 64 111 111 111 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 64 64 64 110 110 110 F F F 6 120-139 64 64 64 111 111 111 F F F 7 140-159 64 64 64 110 110 110 F F F 8 160-179 60 60 60 103 103 103 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 64 64 64 110 110 110 T T T 13 260-279 64 64 64 110 110 110 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 64 64 64 110 110 110 F F F 17 340-359 64 64 64 110 110 110 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 63 63 63 108 108 108 T T T 21 420-439 54 54 54 92 92 92 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 60 60 60 102 102 102 F F F 25 500-519 62 62 62 107 107 107 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 62 62 62 107 107 107 F F F Max gen: 64, Max flush: 111 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 65 65 65 112 112 112 F F F 3 060-079 64 64 64 111 111 111 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 64 64 64 110 110 110 F F F 6 120-139 64 64 64 111 111 111 F F F 7 140-159 64 64 64 110 110 110 F F F 8 160-179 60 60 60 103 103 103 F F F 9 180-199 64 64 64 110 110 110 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 65 65 65 112 112 112 F F F 13 260-279 64 64 64 110 110 110 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 103 103 103 F F F 16 320-339 64 64 64 110 110 110 F F F 17 340-359 64 64 64 110 110 110 F F F 18 360-379 59 59 59 101 101 101 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 65 65 65 112 112 112 F F F 21 420-439 54 54 54 92 92 92 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 60 60 60 102 102 102 F F F 25 500-519 62 62 62 107 107 107 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 62 62 62 107 107 107 F F F Max gen: 65, Max flush: 112 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.862363197Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.906609051Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.907381005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.90742229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.907437647Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.907452405Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} {"msg":"Crucible 48d34160-4aa5-4d99-aa06-9f34192487ea has session id: ace84d8c-c245-44b4-b444-b0f53a4a5cff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908108225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Upstairs opts: Upstairs UUID: 48d34160-4aa5-4d99-aa06-9f34192487ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908156238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Crucible stats registered with UUID: 48d34160-4aa5-4d99-aa06-9f34192487ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908178415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:56.908196443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908612646Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908638642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} {"msg":"48d34160-4aa5-4d99-aa06-9f34192487ea active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908702633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908720781Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"0","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908750617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"0","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908765135Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"1","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908780353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"1","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.9088031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"2","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908829476Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"2","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908855302Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908882989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908908055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:49212)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908933042Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:63878)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908954779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"ds_connection connected from Ok(127.0.0.1:48092)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.908983195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.909713144Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"2","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.909753659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"0","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.910421097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"0","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.910463801Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"2","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.911024844Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.911144827Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.911833223Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"1","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.912378928Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"1","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[0]R flush_numbers[0..12]: [103, 106, 112, 111, 102, 110, 111, 110, 103, 110, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.91287261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[0]R generation[0..12]: [60, 62, 65, 64, 60, 64, 64, 64, 60, 64, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.912898246Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.912914114Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[1]R flush_numbers[0..12]: [103, 106, 112, 111, 102, 110, 111, 110, 103, 110, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.912928952Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[1]R generation[0..12]: [60, 62, 65, 64, 60, 64, 64, 64, 60, 64, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.91294317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.912961198Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[2]R flush_numbers[0..12]: [103, 106, 112, 111, 102, 110, 111, 110, 103, 110, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.912975866Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[2]R generation[0..12]: [60, 62, 65, 64, 60, 64, 64, 64, 60, 64, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.912989954Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913004462Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.9130183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913032118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913045716Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913059034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"":"downstairs","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913072542Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913084881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"0","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913098439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"1","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913112137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"client":"2","":"downstairs","session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:56.913125715Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913146652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"48d34160-4aa5-4d99-aa06-9f34192487ea is now active with session: ace84d8c-c245-44b4-b444-b0f53a4a5cff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.91315978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913173818Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456,"session_id":"ace84d8c-c245-44b4-b444-b0f53a4a5cff"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913186467Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} {"msg":"Activated sub_volume 48d34160-4aa5-4d99-aa06-9f34192487ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:56.913198225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 314 len 12288 data: 5 5 5 02/30 Flush 03/30 Flush 04/30 Write block 378 len 24576 data: 3 3 4 4 4 4 05/30 Read block 121 len 28672 06/30 Read block 406 len 8192 07/30 Write block 273 len 4096 data: 5 08/30 Flush 09/30 Write block 179 len 32768 data: 4 4 5 5 4 4 5 5 10/30 Write block 152 len 36864 data: 6 6 7 8 10 10 8 7 8 11/30 Write block 77 len 12288 data: 8 8 7 12/30 Read block 111 len 16384 13/30 Flush 14/30 Write block 526 len 36864 data: 6 4 5 5 5 5 5 6 7 15/30 Read block 32 len 8192 16/30 Read block 284 len 8192 17/30 Read block 489 len 40960 18/30 Write block 498 len 28672 data: 8 9 8 7 6 7 7 19/30 Read block 437 len 12288 20/30 Read block 405 len 36864 21/30 Write block 59 len 12288 data: 10 9 11 22/30 Flush 23/30 Write block 172 len 40960 data: 6 6 5 5 4 4 4 5 5 6 24/30 Write block 351 len 36864 data: 6 7 7 6 6 8 7 7 6 25/30 Read block 189 len 36864 26/30 Read block 152 len 16384 27/30 Read block 468 len 24576 28/30 Write block 422 len 36864 data: 6 5 6 7 6 6 5 5 5 29/30 Write block 526 len 32768 data: 7 5 6 6 6 6 6 7 30/30 Read block 74 len 16384 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 7 8 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:56.934654064Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 66 66 66 117 117 117 F F F 3 060-079 66 66 66 117 117 117 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 64 64 64 110 110 110 F F F 6 120-139 64 64 64 111 111 111 F F F 7 140-159 66 66 66 116 116 116 F F F 8 160-179 66 66 66 116 116 116 T T T 9 180-199 66 66 66 116 116 116 T T T 10 200-219 52 52 52 90 90 90 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 65 65 65 112 112 112 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 66 66 66 113 113 113 F F F 16 320-339 64 64 64 110 110 110 F F F 17 340-359 64 64 64 110 110 110 T T T 18 360-379 66 66 66 115 115 115 F F F 19 380-399 66 66 66 115 115 115 F F F 20 400-419 65 65 65 112 112 112 F F F 21 420-439 54 54 54 92 92 92 T T T 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 66 66 66 117 117 117 F F F 26 520-539 66 66 66 117 117 117 T T T 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 62 62 62 107 107 107 F F F Max gen: 66, Max flush: 117 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 66 66 66 117 117 117 F F F 3 060-079 66 66 66 117 117 117 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 64 64 64 110 110 110 F F F 6 120-139 64 64 64 111 111 111 F F F 7 140-159 66 66 66 116 116 116 F F F 8 160-179 67 67 67 118 118 118 F F F 9 180-199 67 67 67 118 118 118 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 65 65 65 112 112 112 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 66 66 66 113 113 113 F F F 16 320-339 64 64 64 110 110 110 F F F 17 340-359 67 67 67 118 118 118 F F F 18 360-379 66 66 66 115 115 115 F F F 19 380-399 66 66 66 115 115 115 F F F 20 400-419 65 65 65 112 112 112 F F F 21 420-439 67 67 67 118 118 118 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 66 66 66 117 117 117 F F F 26 520-539 67 67 67 118 118 118 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 62 62 62 107 107 107 F F F Max gen: 67, Max flush: 118 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.184928666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.230239254Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.231038704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.231073499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.231093896Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.231109794Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} {"msg":"Crucible 3858e463-cf6e-4a86-9bcc-1e89079b65f6 has session id: 04fd1c29-98a0-4b00-b804-8d63c39d98eb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.231642871Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Upstairs opts: Upstairs UUID: 3858e463-cf6e-4a86-9bcc-1e89079b65f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.231689524Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Crucible stats registered with UUID: 3858e463-cf6e-4a86-9bcc-1e89079b65f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.231708802Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:01.23172232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232181926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232207363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} {"msg":"3858e463-cf6e-4a86-9bcc-1e89079b65f6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232289462Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232315348Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"0","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232344114Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"0","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232359312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"1","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.23237457Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"1","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232388518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232407235Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232421453Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232435971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232452059Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:47564)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232497433Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46511)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232658981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:01.232793432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:01.232826168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:01.232856454Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.232870862Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.233188118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"1","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.233217794Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"0","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.233851387Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"0","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.233983159Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"1","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.234355247Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:01.234570028Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.233996601Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:35675)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.234136002Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.235219593Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.235814921Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[0]R flush_numbers[0..12]: [103, 106, 117, 117, 102, 110, 111, 116, 118, 118, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236443975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[0]R generation[0..12]: [60, 62, 66, 66, 60, 64, 64, 66, 67, 67, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.23647383Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236493388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[1]R flush_numbers[0..12]: [103, 106, 117, 117, 102, 110, 111, 116, 118, 118, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236529963Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[1]R generation[0..12]: [60, 62, 66, 66, 60, 64, 64, 66, 67, 67, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.23654872Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236566887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[2]R flush_numbers[0..12]: [103, 106, 117, 117, 102, 110, 111, 116, 118, 118, 90, 107]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236584605Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[2]R generation[0..12]: [60, 62, 66, 66, 60, 64, 64, 66, 67, 67, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236602063Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.23662613Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236643217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236660365Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236677003Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.23669379Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"":"downstairs","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236710698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236726106Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"0","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236743123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"1","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:05:11.236760261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"client":"2","":"downstairs","session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236783688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236799526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"3858e463-cf6e-4a86-9bcc-1e89079b65f6 is now active with session: 04fd1c29-98a0-4b00-b804-8d63c39d98eb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236815743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236830911Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464,"session_id":"04fd1c29-98a0-4b00-b804-8d63c39d98eb"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236846589Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} {"msg":"Activated sub_volume 3858e463-cf6e-4a86-9bcc-1e89079b65f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.236860967Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 32768 data: 6 8 9 10 7 9 10 9 02/30 Write block 515 len 8192 data: 11 8 03/30 Flush 04/30 Write block 353 len 28672 data: 8 7 7 9 8 8 7 05/30 Read block 566 len 32768 06/30 Read block 15 len 40960 07/30 Write block 96 len 12288 data: 8 8 7 08/30 Read block 0 len 24576 09/30 Flush 10/30 Write block 588 len 32768 data: 6 5 8 6 6 6 7 6 11/30 Read block 30 len 12288 12/30 Write block 218 len 16384 data: 9 7 9 9 13/30 Write block 142 len 40960 data: 5 6 7 9 9 9 9 8 8 8 14/30 Read block 280 len 32768 15/30 Read block 121 len 24576 16/30 Flush 17/30 Write block 217 len 28672 data: 9 10 8 10 10 9 9 18/30 Read block 537 len 16384 19/30 Read block 224 len 8192 20/30 Write block 344 len 28672 data: 9 8 7 7 6 5 5 21/30 Read block 180 len 36864 22/30 Read block 114 len 40960 23/30 Read block 37 len 16384 24/30 Read block 104 len 4096 25/30 Read block 306 len 28672 26/30 Read block 166 len 16384 27/30 Write block 247 len 16384 data: 7 8 6 6 28/30 Read block 129 len 40960 29/30 Write block 354 len 32768 data: 8 8 10 9 9 8 5 5 30/30 Write block 314 len 40960 data: 6 6 6 4 5 5 4 5 5 6 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Sent Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:11.259975248Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 66 66 66 117 117 117 F F F 3 060-079 66 66 66 117 117 117 F F F 4 080-099 68 68 68 120 120 120 F F F 5 100-119 64 64 64 110 110 110 F F F 6 120-139 64 64 64 111 111 111 F F F 7 140-159 68 68 68 121 121 121 F F F 8 160-179 67 67 67 118 118 118 F F F 9 180-199 67 67 67 118 118 118 F F F 10 200-219 68 68 68 121 121 121 T T T 11 220-239 68 68 68 121 121 121 T T T 12 240-259 68 68 68 119 119 119 T T T 13 260-279 66 66 66 115 115 115 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 66 66 66 113 113 113 T T T 16 320-339 64 64 64 110 110 110 T T T 17 340-359 68 68 68 120 120 120 T T T 18 360-379 66 66 66 115 115 115 T T T 19 380-399 66 66 66 115 115 115 F F F 20 400-419 65 65 65 112 112 112 F F F 21 420-439 67 67 67 118 118 118 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 68 68 68 119 119 119 F F F 26 520-539 67 67 67 118 118 118 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 68 68 68 121 121 121 F F F Max gen: 68, Max flush: 121 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 103 103 103 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 66 66 66 117 117 117 F F F 3 060-079 66 66 66 117 117 117 F F F 4 080-099 68 68 68 120 120 120 F F F 5 100-119 64 64 64 110 110 110 F F F 6 120-139 64 64 64 111 111 111 F F F 7 140-159 68 68 68 121 121 121 F F F 8 160-179 67 67 67 118 118 118 F F F 9 180-199 67 67 67 118 118 118 F F F 10 200-219 69 69 69 122 122 122 F F F 11 220-239 69 69 69 122 122 122 F F F 12 240-259 69 69 69 122 122 122 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 69 69 69 122 122 122 F F F 18 360-379 69 69 69 122 122 122 F F F 19 380-399 66 66 66 115 115 115 F F F 20 400-419 65 65 65 112 112 112 F F F 21 420-439 67 67 67 118 118 118 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 68 68 68 119 119 119 F F F 26 520-539 67 67 67 118 118 118 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 68 68 68 121 121 121 F F F Max gen: 69, Max flush: 122 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.521089448Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:15.5646211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.565010887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.597129679Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.597773671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.597815935Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.597839531Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.597858499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} {"msg":"Crucible ccc30b3f-6373-4978-8cbe-f07d307b3eb0 has session id: f797f88f-60dd-4ed4-acc5-ba78053ab0d6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.598417762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Upstairs opts: Upstairs UUID: ccc30b3f-6373-4978-8cbe-f07d307b3eb0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.598468515Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Crucible stats registered with UUID: ccc30b3f-6373-4978-8cbe-f07d307b3eb0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.598501151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:15.598519638Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599000792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599024468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} {"msg":"ccc30b3f-6373-4978-8cbe-f07d307b3eb0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.59908772Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599106047Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"0","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599134244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"0","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599149411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"1","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599171738Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"1","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599196515Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"2","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599212683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"2","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.59923047Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599246118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599263096Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51259)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599538358Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42946)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599581792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:52433)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.599612418Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.600207256Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"2","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.600236692Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"0","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.60025291Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"1","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.600860186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"1","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601016245Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"0","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601045341Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"2","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.60156046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601688882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[0]R flush_numbers[0..12]: [103, 106, 117, 117, 120, 110, 111, 121, 118, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601714649Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[0]R generation[0..12]: [60, 62, 66, 66, 68, 64, 64, 68, 67, 67, 69, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601730686Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601745844Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[1]R flush_numbers[0..12]: [103, 106, 117, 117, 120, 110, 111, 121, 118, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601760282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[1]R generation[0..12]: [60, 62, 66, 66, 68, 64, 64, 68, 67, 67, 69, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.60177438Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601788888Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[2]R flush_numbers[0..12]: [103, 106, 117, 117, 120, 110, 111, 121, 118, 118, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601803326Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[2]R generation[0..12]: [60, 62, 66, 66, 68, 64, 64, 68, 67, 67, 69, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601817634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601832172Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.60184568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601859329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601872687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601886095Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"":"downstairs","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601899933Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601914181Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"0","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601928699Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"1","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601942587Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"client":"2","":"downstairs","session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601956655Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601969454Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"ccc30b3f-6373-4978-8cbe-f07d307b3eb0 is now active with session: f797f88f-60dd-4ed4-acc5-ba78053ab0d6"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.601989221Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.602001989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472,"session_id":"f797f88f-60dd-4ed4-acc5-ba78053ab0d6"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.602014497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} {"msg":"Activated sub_volume ccc30b3f-6373-4978-8cbe-f07d307b3eb0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.602025986Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 345 len 32768 data: 9 8 8 7 6 6 7 8 02/30 Read block 191 len 28672 03/30 Write block 165 len 36864 data: 6 3 3 4 7 7 7 7 7 04/30 Read block 289 len 24576 05/30 Read block 366 len 36864 06/30 Read block 44 len 36864 07/30 Write block 7 len 32768 data: 4 3 4 6 6 7 8 9 08/30 Read block 337 len 4096 09/30 Write block 367 len 20480 data: 6 6 7 8 6 10/30 Flush 11/30 Write block 17 len 28672 data: 8 9 9 6 5 4 4 12/30 Write block 163 len 12288 data: 8 8 7 13/30 Write block 42 len 28672 data: 12 12 11 10 9 9 6 14/30 Read block 239 len 36864 15/30 Write block 57 len 32768 data: 8 8 11 10 12 11 12 11 16/30 Read block 519 len 4096 17/30 Read block 423 len 28672 18/30 Write block 198 len 28672 data: 4 5 5 5 5 4 4 19/30 Write block 509 len 24576 data: 5 6 7 8 8 11 20/30 Write block 526 len 4096 data: 8 21/30 Read block 558 len 32768 22/30 Flush 23/30 Write block 125 len 24576 data: 7 8 6 6 6 6 24/30 Write block 241 len 40960 data: 10 7 8 8 9 8 8 9 7 7 25/30 Write block 503 len 20480 data: 8 8 6 7 5 26/30 Read block 156 len 28672 27/30 Write block 418 len 8192 data: 6 6 28/30 Write block 115 len 8192 data: 4 5 29/30 Read block 285 len 8192 30/30 Write block 135 len 20480 data: 7 4 3 3 3 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:15.623380998Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 124 124 124 F F F 1 020-039 70 70 70 124 124 124 F F F 2 040-059 70 70 70 124 124 124 F F F 3 060-079 70 70 70 124 124 124 F F F 4 080-099 68 68 68 120 120 120 F F F 5 100-119 64 64 64 110 110 110 T T T 6 120-139 64 64 64 111 111 111 T T T 7 140-159 68 68 68 121 121 121 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 69 69 69 122 122 122 F F F 12 240-259 69 69 69 122 122 122 T T T 13 260-279 66 66 66 115 115 115 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 F F F 19 380-399 66 66 66 115 115 115 F F F 20 400-419 65 65 65 112 112 112 T T T 21 420-439 67 67 67 118 118 118 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 70 70 70 124 124 124 T T T 26 520-539 70 70 70 124 124 124 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 68 68 68 121 121 121 F F F Max gen: 70, Max flush: 124 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 124 124 124 F F F 1 020-039 70 70 70 124 124 124 F F F 2 040-059 70 70 70 124 124 124 F F F 3 060-079 70 70 70 124 124 124 F F F 4 080-099 68 68 68 120 120 120 F F F 5 100-119 71 71 71 125 125 125 F F F 6 120-139 71 71 71 125 125 125 F F F 7 140-159 68 68 68 121 121 121 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 70 70 70 124 124 124 F F F 10 200-219 70 70 70 124 124 124 F F F 11 220-239 69 69 69 122 122 122 F F F 12 240-259 71 71 71 125 125 125 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 F F F 19 380-399 66 66 66 115 115 115 F F F 20 400-419 71 71 71 125 125 125 F F F 21 420-439 67 67 67 118 118 118 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 70 70 70 124 124 124 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 55 55 55 95 95 95 F F F 29 580-599 68 68 68 121 121 121 F F F Max gen: 71, Max flush: 125 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.899511712Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.946035184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.94686343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.946907613Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.946926041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.946944579Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} {"msg":"Crucible 03d7aa02-b491-4f69-a3bb-2c6b1db4c090 has session id: 771dbc6f-6983-4481-b06d-0da9a359f834","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.947473196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Upstairs opts: Upstairs UUID: 03d7aa02-b491-4f69-a3bb-2c6b1db4c090, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.94751577Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Crucible stats registered with UUID: 03d7aa02-b491-4f69-a3bb-2c6b1db4c090","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.947539387Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:19.947559914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948150563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948184598Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} {"msg":"03d7aa02-b491-4f69-a3bb-2c6b1db4c090 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948258088Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948277985Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"0","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948306831Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"0","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948321429Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"1","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948336167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"1","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948356044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"2","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948382821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"2","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948409847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948426795Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948441543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"ds_connection connected from Ok(127.0.0.1:40313)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948743881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"ds_connection connected from Ok(127.0.0.1:43397)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948786705Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"ds_connection connected from Ok(127.0.0.1:63607)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.948803953Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.94940211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"1","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.949431126Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"0","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.949450584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"2","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.949978651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"1","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950039153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"2","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950105704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"0","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950664277Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950694203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[0]R flush_numbers[0..12]: [124, 124, 124, 124, 120, 125, 125, 121, 124, 124, 124, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950722859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[0]R generation[0..12]: [70, 70, 70, 70, 68, 71, 71, 68, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950738917Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950753864Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[1]R flush_numbers[0..12]: [124, 124, 124, 124, 120, 125, 125, 121, 124, 124, 124, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950768582Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[1]R generation[0..12]: [70, 70, 70, 70, 68, 71, 71, 68, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950782891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950797349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[2]R flush_numbers[0..12]: [124, 124, 124, 124, 120, 125, 125, 121, 124, 124, 124, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950811467Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[2]R generation[0..12]: [70, 70, 70, 70, 68, 71, 71, 68, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950828924Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.95085951Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950875338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950889036Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950902874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950916522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"":"downstairs","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.95093038Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950942769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"0","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950956607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"1","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-18T05:05:19.950979533Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"client":"2","":"downstairs","session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.950993512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.95100623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"03d7aa02-b491-4f69-a3bb-2c6b1db4c090 is now active with session: 771dbc6f-6983-4481-b06d-0da9a359f834","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.951019488Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.951031806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480,"session_id":"771dbc6f-6983-4481-b06d-0da9a359f834"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.951044415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} {"msg":"Activated sub_volume 03d7aa02-b491-4f69-a3bb-2c6b1db4c090","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.951056323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 191 len 40960 data: 9 7 6 5 4 4 3 5 6 6 02/30 Write block 236 len 8192 data: 10 11 03/30 Read block 155 len 8192 04/30 Write block 112 len 32768 data: 5 4 3 5 6 5 4 7 05/30 Write block 390 len 12288 data: 4 3 4 06/30 Read block 61 len 24576 07/30 Read block 199 len 8192 08/30 Read block 178 len 36864 09/30 Read block 519 len 36864 10/30 Write block 176 len 32768 data: 5 5 5 6 6 7 6 5 11/30 Write block 294 len 8192 data: 4 4 12/30 Flush 13/30 Write block 405 len 4096 data: 6 14/30 Write block 419 len 24576 data: 7 6 6 7 6 7 15/30 Read block 450 len 8192 16/30 Write block 19 len 8192 data: 10 7 17/30 Read block 140 len 16384 18/30 Write block 572 len 28672 data: 4 3 3 3 3 3 6 19/30 Flush 20/30 Write block 161 len 16384 data: 7 9 9 9 21/30 Flush 22/30 Write block 170 len 24576 data: 8 8 8 8 6 6 23/30 Write block 255 len 20480 data: 8 9 9 10 11 24/30 Flush 25/30 Write block 466 len 28672 data: 5 5 7 6 6 8 8 26/30 Read block 470 len 24576 27/30 Write block 470 len 24576 data: 7 9 9 9 7 7 28/30 Write block 61 len 12288 data: 13 12 13 29/30 Read block 322 len 40960 30/30 Write block 370 len 28672 data: 9 7 6 5 3 3 3 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:19.971591329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 127 127 127 F F F 1 020-039 72 72 72 127 127 127 F F F 2 040-059 70 70 70 124 124 124 F F F 3 060-079 70 70 70 124 124 124 T T T 4 080-099 68 68 68 120 120 120 F F F 5 100-119 72 72 72 126 126 126 F F F 6 120-139 71 71 71 125 125 125 F F F 7 140-159 68 68 68 121 121 121 F F F 8 160-179 72 72 72 129 129 129 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 72 72 72 126 126 126 F F F 11 220-239 72 72 72 126 126 126 F F F 12 240-259 72 72 72 129 129 129 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 72 72 72 126 126 126 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 T T T 19 380-399 72 72 72 126 126 126 F F F 20 400-419 72 72 72 127 127 127 F F F 21 420-439 72 72 72 127 127 127 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 61 61 61 104 104 104 T T T 24 480-499 66 66 66 117 117 117 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 70 70 70 124 124 124 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 68 68 68 121 121 121 F F F Max gen: 72, Max flush: 129 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 127 127 127 F F F 1 020-039 72 72 72 127 127 127 F F F 2 040-059 70 70 70 124 124 124 F F F 3 060-079 73 73 73 130 130 130 F F F 4 080-099 68 68 68 120 120 120 F F F 5 100-119 72 72 72 126 126 126 F F F 6 120-139 71 71 71 125 125 125 F F F 7 140-159 68 68 68 121 121 121 F F F 8 160-179 72 72 72 129 129 129 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 72 72 72 126 126 126 F F F 11 220-239 72 72 72 126 126 126 F F F 12 240-259 72 72 72 129 129 129 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 72 72 72 126 126 126 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 73 73 73 130 130 130 F F F 19 380-399 72 72 72 126 126 126 F F F 20 400-419 72 72 72 127 127 127 F F F 21 420-439 72 72 72 127 127 127 F F F 22 440-459 61 61 61 104 104 104 F F F 23 460-479 73 73 73 130 130 130 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 70 70 70 124 124 124 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 68 68 68 121 121 121 F F F Max gen: 73, Max flush: 130 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.209555604Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.254210262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.255044077Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.255084342Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.255101509Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.255119907Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} {"msg":"Crucible 293b41b9-5b23-44c4-a703-2770a249ab8c has session id: eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.255773097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Upstairs opts: Upstairs UUID: 293b41b9-5b23-44c4-a703-2770a249ab8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.255827159Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Crucible stats registered with UUID: 293b41b9-5b23-44c4-a703-2770a249ab8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.255853456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:24.255875373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256351547Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256377274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} {"msg":"293b41b9-5b23-44c4-a703-2770a249ab8c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256437455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256455503Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"0","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256484549Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"0","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256499177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"1","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256513735Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"1","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256527373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256542441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256556349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256571747Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256598973Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:24.256707668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:24.256749433Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"ds_connection connected from Ok(127.0.0.1:47370)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256778079Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:24.256809154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"ds_connection connected from Ok(127.0.0.1:41182)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256834171Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.256921149Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.257525836Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"1","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.25756647Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"0","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.25822199Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"1","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.258252206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"0","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.258855283Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.258890168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.25826513Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"ds_connection connected from Ok(127.0.0.1:34026)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.258343179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.259497391Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260256516Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[0]R flush_numbers[0..12]: [127, 127, 124, 130, 120, 126, 125, 121, 129, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260863353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[0]R generation[0..12]: [72, 72, 70, 73, 68, 72, 71, 68, 72, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.26088512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260900887Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[1]R flush_numbers[0..12]: [127, 127, 124, 130, 120, 126, 125, 121, 129, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260915456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[1]R generation[0..12]: [72, 72, 70, 73, 68, 72, 71, 68, 72, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260929774Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260944721Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[2]R flush_numbers[0..12]: [127, 127, 124, 130, 120, 126, 125, 121, 129, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260959019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[2]R generation[0..12]: [72, 72, 70, 73, 68, 72, 71, 68, 72, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260973217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.260988015Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261001384Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261015152Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261032619Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261046487Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"":"downstairs","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261060416Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261072874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"0","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261087212Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"1","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.26110119Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"client":"2","":"downstairs","session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261115128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261127666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"293b41b9-5b23-44c4-a703-2770a249ab8c is now active with session: eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261140805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261153093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488,"session_id":"eb01cbce-8f2e-4fbb-bdb9-91ec4f872e87"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.261165801Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} {"msg":"Activated sub_volume 293b41b9-5b23-44c4-a703-2770a249ab8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:34.26117744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 40960 data: 5 5 5 5 6 6 6 6 5 7 02/30 Read block 254 len 4096 03/30 Write block 84 len 20480 data: 6 6 7 6 8 04/30 Write block 31 len 12288 data: 3 3 3 05/30 Read block 178 len 4096 06/30 Write block 529 len 28672 data: 7 7 7 7 8 8 7 07/30 Write block 124 len 12288 data: 7 8 9 08/30 Write block 115 len 20480 data: 6 7 6 5 8 09/30 Flush 10/30 Write block 145 len 16384 data: 10 10 10 10 11/30 Read block 579 len 28672 12/30 Read block 62 len 4096 13/30 Write block 10 len 28672 data: 7 7 8 9 10 7 7 14/30 Flush 15/30 Write block 587 len 32768 data: 7 7 6 9 7 7 7 8 16/30 Read block 135 len 12288 17/30 Write block 441 len 28672 data: 5 5 7 6 8 8 11 18/30 Write block 465 len 4096 data: 4 19/30 Write block 429 len 12288 data: 6 6 4 20/30 Write block 102 len 16384 data: 6 8 8 7 21/30 Flush 22/30 Write block 525 len 16384 data: 7 9 6 7 23/30 Read block 293 len 12288 24/30 Read block 154 len 36864 25/30 Write block 344 len 8192 data: 10 10 26/30 Read block 408 len 12288 27/30 Read block 102 len 24576 28/30 Read block 142 len 4096 29/30 Write block 103 len 24576 data: 9 9 8 5 7 8 30/30 Read block 375 len 8192 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:34.281182218Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 74 74 74 131 131 131 F F F 2 040-059 70 70 70 124 124 124 F F F 3 060-079 73 73 73 130 130 130 F F F 4 080-099 74 74 74 131 131 131 F F F 5 100-119 74 74 74 133 133 133 T T T 6 120-139 74 74 74 131 131 131 F F F 7 140-159 74 74 74 132 132 132 F F F 8 160-179 72 72 72 129 129 129 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 72 72 72 126 126 126 F F F 11 220-239 72 72 72 126 126 126 F F F 12 240-259 72 72 72 129 129 129 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 72 72 72 126 126 126 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 70 70 70 123 123 123 T T T 18 360-379 73 73 73 130 130 130 F F F 19 380-399 72 72 72 126 126 126 F F F 20 400-419 74 74 74 131 131 131 F F F 21 420-439 74 74 74 133 133 133 F F F 22 440-459 74 74 74 133 133 133 F F F 23 460-479 74 74 74 133 133 133 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 74 74 74 131 131 131 T T T 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 74, Max flush: 133 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 74 74 74 131 131 131 F F F 2 040-059 70 70 70 124 124 124 F F F 3 060-079 73 73 73 130 130 130 F F F 4 080-099 74 74 74 131 131 131 F F F 5 100-119 75 75 75 134 134 134 F F F 6 120-139 74 74 74 131 131 131 F F F 7 140-159 74 74 74 132 132 132 F F F 8 160-179 72 72 72 129 129 129 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 72 72 72 126 126 126 F F F 11 220-239 72 72 72 126 126 126 F F F 12 240-259 72 72 72 129 129 129 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 72 72 72 126 126 126 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 75 75 75 134 134 134 F F F 18 360-379 73 73 73 130 130 130 F F F 19 380-399 72 72 72 126 126 126 F F F 20 400-419 74 74 74 131 131 131 F F F 21 420-439 74 74 74 133 133 133 F F F 22 440-459 74 74 74 133 133 133 F F F 23 460-479 74 74 74 133 133 133 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 75 75 75 134 134 134 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 75, Max flush: 134 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.516952435Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.561273089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.562058871Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.562102945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.562120612Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.562155507Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} {"msg":"Crucible ea40b30b-9417-41a2-8ac6-cecfdff0b575 has session id: bab3c5e7-13a5-4194-9274-ef77634c8985","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.562823186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Upstairs opts: Upstairs UUID: ea40b30b-9417-41a2-8ac6-cecfdff0b575, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.562871359Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Crucible stats registered with UUID: ea40b30b-9417-41a2-8ac6-cecfdff0b575","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.562895116Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:38.562909524Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563479756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563524119Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} {"msg":"ea40b30b-9417-41a2-8ac6-cecfdff0b575 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563631075Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563655221Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"0","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563688417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"0","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563705334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"1","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563721752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"1","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.56373824Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"2","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563753868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"2","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563767466Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563782124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563796902Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"ds_connection connected from Ok(127.0.0.1:59813)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563928334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"ds_connection connected from Ok(127.0.0.1:47764)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.563997704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"ds_connection connected from Ok(127.0.0.1:49850)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.564037019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.564791975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"0","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.564833349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"1","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.565474281Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"1","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.565546881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"0","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.566175625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.566201461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.566336883Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"2","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.566849412Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"2","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[0]R flush_numbers[0..12]: [132, 131, 124, 130, 131, 134, 131, 132, 129, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567361491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[0]R generation[0..12]: [74, 74, 70, 73, 74, 75, 74, 74, 72, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567384068Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567399936Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[1]R flush_numbers[0..12]: [132, 131, 124, 130, 131, 134, 131, 132, 129, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567414494Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[1]R generation[0..12]: [74, 74, 70, 73, 74, 75, 74, 74, 72, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567429042Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.56744811Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[2]R flush_numbers[0..12]: [132, 131, 124, 130, 131, 134, 131, 132, 129, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567463088Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[2]R generation[0..12]: [74, 74, 70, 73, 74, 75, 74, 74, 72, 72, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567477166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567491394Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567506691Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567530838Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567560764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567582981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"":"downstairs","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567606488Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567629525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"0","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567654511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"1","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567678968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"client":"2","":"downstairs","session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567695436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567715693Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"ea40b30b-9417-41a2-8ac6-cecfdff0b575 is now active with session: bab3c5e7-13a5-4194-9274-ef77634c8985","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:05:38.567741139Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567768006Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496,"session_id":"bab3c5e7-13a5-4194-9274-ef77634c8985"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567784124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} {"msg":"Activated sub_volume ea40b30b-9417-41a2-8ac6-cecfdff0b575","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:38.567796432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 99 len 12288 data: 10 9 8 02/30 Read block 27 len 4096 03/30 Write block 285 len 12288 data: 4 5 6 04/30 Read block 543 len 8192 05/30 Write block 342 len 12288 data: 8 8 11 06/30 Read block 378 len 4096 07/30 Read block 348 len 20480 08/30 Read block 293 len 12288 09/30 Write block 444 len 4096 data: 7 10/30 Flush 11/30 Write block 44 len 32768 data: 12 11 10 10 7 4 4 6 12/30 Write block 118 len 36864 data: 6 9 8 10 9 7 8 9 10 13/30 Write block 84 len 40960 data: 7 7 8 7 9 8 9 9 10 10 14/30 Write block 215 len 4096 data: 8 15/30 Read block 538 len 8192 16/30 Read block 285 len 24576 17/30 Flush 18/30 Write block 86 len 36864 data: 9 8 10 9 10 10 11 11 9 19/30 Read block 350 len 8192 20/30 Read block 259 len 36864 21/30 Read block 258 len 28672 22/30 Read block 13 len 12288 23/30 Read block 263 len 4096 24/30 Read block 506 len 4096 25/30 Read block 140 len 36864 26/30 Read block 437 len 16384 27/30 Read block 199 len 4096 28/30 Write block 242 len 28672 data: 8 9 9 10 9 9 10 29/30 Write block 348 len 16384 data: 8 7 7 8 30/30 Read block 595 len 4096 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 13 11 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:38.587215881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 74 74 74 131 131 131 F F F 2 040-059 76 76 76 136 136 136 F F F 3 060-079 73 73 73 130 130 130 F F F 4 080-099 76 76 76 136 136 136 T T T 5 100-119 76 76 76 136 136 136 F F F 6 120-139 76 76 76 136 136 136 F F F 7 140-159 74 74 74 132 132 132 F F F 8 160-179 72 72 72 129 129 129 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 76 76 76 136 136 136 F F F 11 220-239 72 72 72 126 126 126 F F F 12 240-259 72 72 72 129 129 129 T T T 13 260-279 66 66 66 115 115 115 F F F 14 280-299 76 76 76 135 135 135 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 76 76 76 135 135 135 T T T 18 360-379 73 73 73 130 130 130 F F F 19 380-399 72 72 72 126 126 126 F F F 20 400-419 74 74 74 131 131 131 F F F 21 420-439 74 74 74 133 133 133 F F F 22 440-459 76 76 76 135 135 135 F F F 23 460-479 74 74 74 133 133 133 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 75 75 75 134 134 134 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 76, Max flush: 136 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 74 74 74 131 131 131 F F F 2 040-059 76 76 76 136 136 136 F F F 3 060-079 73 73 73 130 130 130 F F F 4 080-099 77 77 77 137 137 137 F F F 5 100-119 76 76 76 136 136 136 F F F 6 120-139 76 76 76 136 136 136 F F F 7 140-159 74 74 74 132 132 132 F F F 8 160-179 72 72 72 129 129 129 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 76 76 76 136 136 136 F F F 11 220-239 72 72 72 126 126 126 F F F 12 240-259 77 77 77 137 137 137 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 76 76 76 135 135 135 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 77 77 77 137 137 137 F F F 18 360-379 73 73 73 130 130 130 F F F 19 380-399 72 72 72 126 126 126 F F F 20 400-419 74 74 74 131 131 131 F F F 21 420-439 74 74 74 133 133 133 F F F 22 440-459 76 76 76 135 135 135 F F F 23 460-479 74 74 74 133 133 133 F F F 24 480-499 66 66 66 117 117 117 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 75 75 75 134 134 134 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 77, Max flush: 137 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.819910828Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.864447572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.865313363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.865351348Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.865370255Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.865386493Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} {"msg":"Crucible 1272841b-1470-47a8-96b4-764f9e31316c has session id: 9ebb94d7-7309-4f7f-8e09-486404652582","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866077678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Upstairs opts: Upstairs UUID: 1272841b-1470-47a8-96b4-764f9e31316c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866122852Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Crucible stats registered with UUID: 1272841b-1470-47a8-96b4-764f9e31316c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866146688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:42.866166716Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866628932Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866664907Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} {"msg":"1272841b-1470-47a8-96b4-764f9e31316c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866710601Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866727299Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"0","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866754405Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"0","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866769473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"1","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866784231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"1","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866802318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"2","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866817096Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"2","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866830474Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866850352Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866879148Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"ds_connection connected from Ok(127.0.0.1:44150)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866909284Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"ds_connection connected from Ok(127.0.0.1:46651)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.866939819Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"ds_connection connected from Ok(127.0.0.1:33280)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.867044675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.867629055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"0","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.867669139Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"1","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.868282465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"1","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.868312021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"0","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.868887322Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.869013964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.869289806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"2","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.869802926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"2","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[0]R flush_numbers[0..12]: [132, 131, 136, 130, 137, 136, 136, 132, 129, 126, 136, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870301437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[0]R generation[0..12]: [74, 74, 76, 73, 77, 76, 76, 74, 72, 72, 76, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870324064Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870339782Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[1]R flush_numbers[0..12]: [132, 131, 136, 130, 137, 136, 136, 132, 129, 126, 136, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.87035468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[1]R generation[0..12]: [74, 74, 76, 73, 77, 76, 76, 74, 72, 72, 76, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870369128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870383856Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[2]R flush_numbers[0..12]: [132, 131, 136, 130, 137, 136, 136, 132, 129, 126, 136, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870398364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[2]R generation[0..12]: [74, 74, 76, 73, 77, 76, 76, 74, 72, 72, 76, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870412452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.87042682Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870440348Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870454046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870474073Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870508748Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"":"downstairs","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870524177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870537015Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"0","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870551003Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"1","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870564581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"client":"2","":"downstairs","session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870578089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870590617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"1272841b-1470-47a8-96b4-764f9e31316c is now active with session: 9ebb94d7-7309-4f7f-8e09-486404652582","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870603926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870616234Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504,"session_id":"9ebb94d7-7309-4f7f-8e09-486404652582"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870628762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} {"msg":"Activated sub_volume 1272841b-1470-47a8-96b4-764f9e31316c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:42.870640301Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 28672 data: 4 5 4 6 7 8 9 02/30 Read block 490 len 16384 03/30 Write block 448 len 8192 data: 12 12 04/30 Write block 422 len 20480 data: 8 7 8 8 7 05/30 Read block 579 len 20480 06/30 Flush 07/30 Write block 484 len 32768 data: 8 8 6 5 4 5 6 5 08/30 Write block 152 len 32768 data: 7 7 8 9 11 11 9 8 09/30 Read block 152 len 16384 10/30 Flush 11/30 Write block 107 len 28672 data: 8 9 9 8 7 6 5 12/30 Write block 477 len 12288 data: 7 5 6 13/30 Write block 484 len 4096 data: 9 14/30 Read block 34 len 20480 15/30 Write block 376 len 4096 data: 4 16/30 Read block 467 len 20480 17/30 Read block 502 len 20480 18/30 Write block 202 len 32768 data: 6 5 5 4 6 6 6 7 19/30 Write block 125 len 28672 data: 10 11 7 7 7 7 6 20/30 Write block 385 len 28672 data: 4 5 5 6 4 5 4 21/30 Read block 61 len 8192 22/30 Read block 530 len 28672 23/30 Read block 360 len 16384 24/30 Write block 228 len 20480 data: 9 6 8 6 8 25/30 Read block 380 len 40960 26/30 Read block 585 len 24576 27/30 Write block 166 len 16384 data: 4 4 5 8 28/30 Write block 382 len 4096 data: 5 29/30 Write block 104 len 12288 data: 10 9 6 30/30 Write block 380 len 16384 data: 5 5 6 5 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:42.889206797Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 74 74 74 131 131 131 F F F 2 040-059 76 76 76 136 136 136 F F F 3 060-079 73 73 73 130 130 130 F F F 4 080-099 77 77 77 137 137 137 F F F 5 100-119 76 76 76 136 136 136 T T T 6 120-139 76 76 76 136 136 136 T T T 7 140-159 78 78 78 139 139 139 F F F 8 160-179 72 72 72 129 129 129 T T T 9 180-199 72 72 72 126 126 126 F F F 10 200-219 76 76 76 136 136 136 T T T 11 220-239 72 72 72 126 126 126 T T T 12 240-259 77 77 77 137 137 137 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 76 76 76 135 135 135 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 77 77 77 137 137 137 F F F 18 360-379 73 73 73 130 130 130 T T T 19 380-399 72 72 72 126 126 126 T T T 20 400-419 74 74 74 131 131 131 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 78 78 78 138 138 138 F F F 23 460-479 74 74 74 133 133 133 T T T 24 480-499 78 78 78 139 139 139 T T T 25 500-519 71 71 71 125 125 125 F F F 26 520-539 75 75 75 134 134 134 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 78, Max flush: 139 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 74 74 74 131 131 131 F F F 2 040-059 76 76 76 136 136 136 F F F 3 060-079 73 73 73 130 130 130 F F F 4 080-099 77 77 77 137 137 137 F F F 5 100-119 79 79 79 140 140 140 F F F 6 120-139 79 79 79 140 140 140 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 79 79 79 140 140 140 F F F 9 180-199 72 72 72 126 126 126 F F F 10 200-219 79 79 79 140 140 140 F F F 11 220-239 79 79 79 140 140 140 F F F 12 240-259 77 77 77 137 137 137 F F F 13 260-279 66 66 66 115 115 115 F F F 14 280-299 76 76 76 135 135 135 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 69 69 69 122 122 122 F F F 17 340-359 77 77 77 137 137 137 F F F 18 360-379 79 79 79 140 140 140 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 74 74 74 131 131 131 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 78 78 78 138 138 138 F F F 23 460-479 79 79 79 140 140 140 F F F 24 480-499 79 79 79 140 140 140 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 75 75 75 134 134 134 F F F 27 540-559 63 63 63 108 108 108 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 79, Max flush: 140 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.160250868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:57.203224417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.203647739Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.235187111Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.235836822Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.235871927Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.235886265Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.235900833Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} {"msg":"Crucible 85952c62-5976-4259-9afe-ed5bbab66c60 has session id: 294c4f66-7e04-427c-bcfb-b51f537be679","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.236510139Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Upstairs opts: Upstairs UUID: 85952c62-5976-4259-9afe-ed5bbab66c60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.236560512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Crucible stats registered with UUID: 85952c62-5976-4259-9afe-ed5bbab66c60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.236585319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:57.236599727Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237025138Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237049785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} {"msg":"85952c62-5976-4259-9afe-ed5bbab66c60 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237122115Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237140842Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"0","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237170698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"0","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237185506Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"1","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237200674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"1","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237215022Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"2","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237295661Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"2","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237352573Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237400607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.23744879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"ds_connection connected from Ok(127.0.0.1:55204)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237494044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"ds_connection connected from Ok(127.0.0.1:39620)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237530649Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"ds_connection connected from Ok(127.0.0.1:45991)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.237566484Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.238410168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"1","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.238440634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"2","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.238643026Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"0","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239083325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"1","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239111681Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"2","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239363667Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"0","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239698981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239805236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[0]R flush_numbers[0..12]: [132, 131, 136, 130, 137, 140, 140, 139, 140, 126, 140, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.23991792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[0]R generation[0..12]: [74, 74, 76, 73, 77, 79, 79, 78, 79, 72, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239938608Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239953946Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[1]R flush_numbers[0..12]: [132, 131, 136, 130, 137, 140, 140, 139, 140, 126, 140, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239971603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[1]R generation[0..12]: [74, 74, 76, 73, 77, 79, 79, 78, 79, 72, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.239985951Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240000319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[2]R flush_numbers[0..12]: [132, 131, 136, 130, 137, 140, 140, 139, 140, 126, 140, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240014577Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[2]R generation[0..12]: [74, 74, 76, 73, 77, 79, 79, 78, 79, 72, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240028515Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240042943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240056392Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:57.240079478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240106574Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240122522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"":"downstairs","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.24013651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240157787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"0","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240182904Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"1","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240206791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"client":"2","":"downstairs","session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240232727Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240252325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 85952c62-5976-4259-9afe-ed5bbab66c60 is now active with session: 294c4f66-7e04-427c-bcfb-b51f537be679","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240278441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240300878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512,"session_id":"294c4f66-7e04-427c-bcfb-b51f537be679"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240315046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} {"msg":"Activated sub_volume 85952c62-5976-4259-9afe-ed5bbab66c60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:57.240326674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 36864 data: 7 5 6 9 9 9 7 10 11 02/30 Write block 36 len 36864 data: 3 2 3 7 9 11 13 13 13 03/30 Read block 157 len 24576 04/30 Write block 53 len 16384 data: 7 7 9 8 05/30 Write block 137 len 8192 data: 4 4 06/30 Write block 55 len 36864 data: 10 9 9 9 12 11 14 13 14 07/30 Write block 413 len 20480 data: 7 7 7 7 6 08/30 Read block 427 len 20480 09/30 Read block 227 len 8192 10/30 Write block 124 len 4096 data: 9 11/30 Write block 104 len 24576 data: 11 10 7 9 10 10 12/30 Write block 165 len 32768 data: 8 5 5 6 9 9 9 9 13/30 Read block 105 len 28672 14/30 Read block 10 len 36864 15/30 Flush 16/30 Write block 555 len 12288 data: 8 7 7 17/30 Write block 179 len 32768 data: 7 7 8 7 6 5 6 6 18/30 Read block 163 len 4096 19/30 Write block 453 len 24576 data: 7 5 4 4 5 6 20/30 Write block 469 len 20480 data: 7 8 10 10 10 21/30 Flush 22/30 Write block 232 len 8192 data: 9 8 23/30 Write block 311 len 16384 data: 4 5 5 7 24/30 Read block 405 len 8192 25/30 Read block 274 len 32768 26/30 Write block 412 len 32768 data: 6 8 8 8 8 7 8 8 27/30 Write block 324 len 36864 data: 6 7 5 5 6 6 4 4 4 28/30 Read block 459 len 20480 29/30 Write block 260 len 36864 data: 10 8 9 8 8 7 6 5 5 30/30 Write block 446 len 40960 data: 9 12 13 13 11 10 8 8 6 5 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:57.260418511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 80 80 80 141 141 141 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 80 80 80 141 141 141 F F F 4 080-099 77 77 77 137 137 137 F F F 5 100-119 80 80 80 141 141 141 F F F 6 120-139 80 80 80 141 141 141 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 80 80 80 142 142 142 F F F 10 200-219 80 80 80 141 141 141 F F F 11 220-239 79 79 79 140 140 140 T T T 12 240-259 77 77 77 137 137 137 F F F 13 260-279 66 66 66 115 115 115 T T T 14 280-299 76 76 76 135 135 135 F F F 15 300-319 69 69 69 122 122 122 T T T 16 320-339 69 69 69 122 122 122 T T T 17 340-359 77 77 77 137 137 137 F F F 18 360-379 79 79 79 140 140 140 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 80 80 80 141 141 141 T T T 21 420-439 78 78 78 138 138 138 F F F 22 440-459 80 80 80 142 142 142 T T T 23 460-479 80 80 80 142 142 142 F F F 24 480-499 79 79 79 140 140 140 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 75 75 75 134 134 134 F F F 27 540-559 80 80 80 142 142 142 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 80, Max flush: 142 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 80 80 80 141 141 141 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 80 80 80 141 141 141 F F F 4 080-099 77 77 77 137 137 137 F F F 5 100-119 80 80 80 141 141 141 F F F 6 120-139 80 80 80 141 141 141 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 80 80 80 142 142 142 F F F 10 200-219 80 80 80 141 141 141 F F F 11 220-239 81 81 81 143 143 143 F F F 12 240-259 77 77 77 137 137 137 F F F 13 260-279 81 81 81 143 143 143 F F F 14 280-299 76 76 76 135 135 135 F F F 15 300-319 81 81 81 143 143 143 F F F 16 320-339 81 81 81 143 143 143 F F F 17 340-359 77 77 77 137 137 137 F F F 18 360-379 79 79 79 140 140 140 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 81 81 81 143 143 143 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 81 81 81 143 143 143 F F F 23 460-479 80 80 80 142 142 142 F F F 24 480-499 79 79 79 140 140 140 F F F 25 500-519 71 71 71 125 125 125 F F F 26 520-539 75 75 75 134 134 134 F F F 27 540-559 80 80 80 142 142 142 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 81, Max flush: 143 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.546286655Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:01.58944519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.589865482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.621411453Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.622091229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.622133814Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.622150881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.622169138Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} {"msg":"Crucible 2ec91337-0132-43a0-9fdc-616664980635 has session id: d3f0b998-e309-4e2f-98a0-9baeaa2d39e7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.622840016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Upstairs opts: Upstairs UUID: 2ec91337-0132-43a0-9fdc-616664980635, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.622875741Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Crucible stats registered with UUID: 2ec91337-0132-43a0-9fdc-616664980635","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.622910407Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:01.622923745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623328879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623350716Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} {"msg":"2ec91337-0132-43a0-9fdc-616664980635 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623426296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623449193Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"0","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623478229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"0","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623492756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"1","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623507525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"1","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623521392Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"2","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623535801Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"2","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623587654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623699028Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623742642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:41970)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623773538Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:59104)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.623800594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48798)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.62383063Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.62448578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"1","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.624514466Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"2","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.624538563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"0","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.62521533Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"1","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.625244886Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"0","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.625261833Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"2","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.625799919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.625911694Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[0]R flush_numbers[0..12]: [132, 141, 141, 141, 137, 141, 141, 139, 142, 142, 141, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.625955978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[0]R generation[0..12]: [74, 80, 80, 80, 77, 80, 80, 78, 80, 80, 80, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.625974825Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.625990163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[1]R flush_numbers[0..12]: [132, 141, 141, 141, 137, 141, 141, 139, 142, 142, 141, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626004931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[1]R generation[0..12]: [74, 80, 80, 80, 77, 80, 80, 78, 80, 80, 80, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626019109Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626033767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[2]R flush_numbers[0..12]: [132, 141, 141, 141, 137, 141, 141, 139, 142, 142, 141, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626048675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[2]R generation[0..12]: [74, 80, 80, 80, 77, 80, 80, 78, 80, 80, 80, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626063133Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.62608076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626094709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626108527Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626121975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626140432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"":"downstairs","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:01.626165409Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626181717Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"0","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626196005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"1","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626209703Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"client":"2","":"downstairs","session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.62622918Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626251757Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"2ec91337-0132-43a0-9fdc-616664980635 is now active with session: d3f0b998-e309-4e2f-98a0-9baeaa2d39e7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626266585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626278943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520,"session_id":"d3f0b998-e309-4e2f-98a0-9baeaa2d39e7"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626293201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} {"msg":"Activated sub_volume 2ec91337-0132-43a0-9fdc-616664980635","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:01.626313648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 12288 data: 8 5 5 02/30 Flush 03/30 Write block 229 len 32768 data: 7 9 7 10 9 7 9 11 04/30 Write block 352 len 16384 data: 9 9 9 9 05/30 Write block 547 len 24576 data: 6 7 7 7 8 9 06/30 Read block 376 len 8192 07/30 Read block 349 len 20480 08/30 Flush 09/30 Write block 290 len 4096 data: 6 10/30 Write block 510 len 4096 data: 7 11/30 Write block 408 len 32768 data: 6 6 6 6 7 9 9 9 12/30 Flush 13/30 Write block 196 len 40960 data: 5 4 6 7 7 6 7 6 6 5 14/30 Write block 73 len 36864 data: 5 3 4 8 9 9 8 7 9 15/30 Read block 418 len 32768 16/30 Read block 400 len 12288 17/30 Write block 80 len 32768 data: 8 10 6 6 8 8 10 9 18/30 Write block 119 len 28672 data: 10 9 11 10 8 10 11 19/30 Write block 462 len 20480 data: 3 3 4 5 6 20/30 Read block 270 len 8192 21/30 Write block 372 len 20480 data: 7 6 4 4 5 22/30 Read block 404 len 36864 23/30 Read block 151 len 36864 24/30 Write block 529 len 16384 data: 8 8 8 8 25/30 Flush 26/30 Flush 27/30 Write block 206 len 12288 data: 7 7 7 28/30 Write block 90 len 32768 data: 11 11 12 12 10 8 9 9 29/30 Write block 526 len 28672 data: 10 7 8 9 9 9 9 30/30 Read block 494 len 40960 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 0 2 Done 4 2 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:01.646417454Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 80 80 80 141 141 141 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 82 82 82 147 147 147 T T T 5 100-119 82 82 82 147 147 147 F F F 6 120-139 82 82 82 147 147 147 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 82 82 82 147 147 147 F F F 10 200-219 82 82 82 147 147 147 T T T 11 220-239 82 82 82 145 145 145 F F F 12 240-259 77 77 77 137 137 137 F F F 13 260-279 81 81 81 143 143 143 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 81 81 81 143 143 143 F F F 16 320-339 81 81 81 143 143 143 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 82 82 82 146 146 146 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 81 81 81 143 143 143 F F F 23 460-479 82 82 82 147 147 147 F F F 24 480-499 79 79 79 140 140 140 F F F 25 500-519 82 82 82 146 146 146 F F F 26 520-539 82 82 82 147 147 147 T T T 27 540-559 82 82 82 145 145 145 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 82, Max flush: 147 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 80 80 80 141 141 141 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 83 83 83 148 148 148 F F F 5 100-119 82 82 82 147 147 147 F F F 6 120-139 82 82 82 147 147 147 F F F 7 140-159 78 78 78 139 139 139 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 82 82 82 147 147 147 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 82 82 82 145 145 145 F F F 12 240-259 77 77 77 137 137 137 F F F 13 260-279 81 81 81 143 143 143 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 81 81 81 143 143 143 F F F 16 320-339 81 81 81 143 143 143 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 82 82 82 146 146 146 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 81 81 81 143 143 143 F F F 23 460-479 82 82 82 147 147 147 F F F 24 480-499 79 79 79 140 140 140 F F F 25 500-519 82 82 82 146 146 146 F F F 26 520-539 83 83 83 148 148 148 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 72 72 72 127 127 127 F F F 29 580-599 74 74 74 133 133 133 F F F Max gen: 83, Max flush: 148 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.913021199Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:05.957309318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.957867261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.989969436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.990749118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.990783354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.990797112Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.99081201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} {"msg":"Crucible 5b1e5c96-0f6a-4a0c-9ae0-16dbbb58a65e has session id: f15a398d-4ae6-4292-853a-bee61cf23bec","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.991482108Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Upstairs opts: Upstairs UUID: 5b1e5c96-0f6a-4a0c-9ae0-16dbbb58a65e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.991525762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Crucible stats registered with UUID: 5b1e5c96-0f6a-4a0c-9ae0-16dbbb58a65e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.991543939Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:05.991558027Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.991926836Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.991950643Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} {"msg":"5b1e5c96-0f6a-4a0c-9ae0-16dbbb58a65e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992017654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.9920447Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"0","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992074986Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"0","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992090594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"1","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992105952Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"1","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.99211978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"2","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992134388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"2","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992151945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992181041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992206868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"ds_connection connected from Ok(127.0.0.1:36861)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992232074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"ds_connection connected from Ok(127.0.0.1:49281)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992259281Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"ds_connection connected from Ok(127.0.0.1:59566)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992285867Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992854639Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"2","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.992997919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"1","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.993091066Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"0","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.993587528Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"2","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.993625823Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"1","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.99379222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"0","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994215852Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994242298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[0]R flush_numbers[0..12]: [132, 141, 141, 147, 148, 147, 147, 139, 142, 147, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994347654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[0]R generation[0..12]: [74, 80, 80, 82, 83, 82, 82, 78, 80, 82, 83, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994368491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994387528Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[1]R flush_numbers[0..12]: [132, 141, 141, 147, 148, 147, 147, 139, 142, 147, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994402446Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[1]R generation[0..12]: [74, 80, 80, 82, 83, 82, 82, 78, 80, 82, 83, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994416764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994431372Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[2]R flush_numbers[0..12]: [132, 141, 141, 147, 148, 147, 147, 139, 142, 147, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.99444569Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[2]R generation[0..12]: [74, 80, 80, 82, 83, 82, 82, 78, 80, 82, 83, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994459778Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994474276Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994487804Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994560514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994579402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.9945931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"":"downstairs","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994606988Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994619416Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"0","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994633034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"1","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994649282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"client":"2","":"downstairs","session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.99466313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994675688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"5b1e5c96-0f6a-4a0c-9ae0-16dbbb58a65e is now active with session: f15a398d-4ae6-4292-853a-bee61cf23bec","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994688836Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994701155Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528,"session_id":"f15a398d-4ae6-4292-853a-bee61cf23bec"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994713583Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} {"msg":"Activated sub_volume 5b1e5c96-0f6a-4a0c-9ae0-16dbbb58a65e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:05.994725351Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 103 len 16384 data: 10 12 11 8 02/30 Flush 03/30 Write block 577 len 24576 data: 4 7 7 7 6 6 04/30 Read block 502 len 40960 05/30 Flush 06/30 Write block 27 len 4096 data: 4 07/30 Read block 287 len 28672 08/30 Write block 247 len 12288 data: 10 11 8 09/30 Read block 125 len 4096 10/30 Flush 11/30 Write block 248 len 32768 data: 12 9 8 4 4 4 6 9 12/30 Read block 165 len 24576 13/30 Write block 116 len 24576 data: 8 7 7 11 10 12 14/30 Read block 194 len 36864 15/30 Read block 286 len 12288 16/30 Read block 37 len 4096 17/30 Read block 49 len 40960 18/30 Write block 520 len 24576 data: 7 7 7 6 6 8 19/30 Read block 476 len 12288 20/30 Write block 149 len 32768 data: 9 9 9 8 8 9 10 12 21/30 Flush 22/30 Write block 443 len 40960 data: 8 8 9 10 13 14 14 12 11 9 23/30 Read block 433 len 4096 24/30 Read block 103 len 40960 25/30 Write block 28 len 32768 data: 3 4 3 4 4 4 2 2 26/30 Read block 306 len 12288 27/30 Read block 463 len 40960 28/30 Read block 29 len 32768 29/30 Write block 480 len 24576 data: 6 5 5 7 10 9 30/30 Read block 252 len 32768 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 9 8 7 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:06.017382445Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 84 84 84 151 151 151 T T T 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 83 83 83 148 148 148 F F F 5 100-119 84 84 84 152 152 152 F F F 6 120-139 84 84 84 152 152 152 F F F 7 140-159 84 84 84 152 152 152 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 82 82 82 147 147 147 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 82 82 82 145 145 145 F F F 12 240-259 84 84 84 152 152 152 F F F 13 260-279 81 81 81 143 143 143 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 81 81 81 143 143 143 F F F 16 320-339 81 81 81 143 143 143 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 82 82 82 146 146 146 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 81 81 81 143 143 143 T T T 23 460-479 82 82 82 147 147 147 F F F 24 480-499 79 79 79 140 140 140 T T T 25 500-519 82 82 82 146 146 146 F F F 26 520-539 84 84 84 152 152 152 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 84 84 84 150 150 150 F F F 29 580-599 84 84 84 150 150 150 F F F Max gen: 84, Max flush: 152 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 83 83 83 148 148 148 F F F 5 100-119 84 84 84 152 152 152 F F F 6 120-139 84 84 84 152 152 152 F F F 7 140-159 84 84 84 152 152 152 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 82 82 82 147 147 147 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 82 82 82 145 145 145 F F F 12 240-259 84 84 84 152 152 152 F F F 13 260-279 81 81 81 143 143 143 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 81 81 81 143 143 143 F F F 16 320-339 81 81 81 143 143 143 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 82 82 82 146 146 146 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 85 85 85 153 153 153 F F F 23 460-479 82 82 82 147 147 147 F F F 24 480-499 85 85 85 153 153 153 F F F 25 500-519 82 82 82 146 146 146 F F F 26 520-539 84 84 84 152 152 152 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 84 84 84 150 150 150 F F F 29 580-599 84 84 84 150 150 150 F F F Max gen: 85, Max flush: 153 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.275108981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.319635617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.320429468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.320467953Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.320482711Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.320497458Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} {"msg":"Crucible a111a42f-f08b-41e2-8179-05d708b815cf has session id: 4d7d30f1-f9f7-460e-872d-8c09d4030a20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.32106881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Upstairs opts: Upstairs UUID: a111a42f-f08b-41e2-8179-05d708b815cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321113094Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Crucible stats registered with UUID: a111a42f-f08b-41e2-8179-05d708b815cf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321136361Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:10.321156928Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321608036Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321640671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} {"msg":"a111a42f-f08b-41e2-8179-05d708b815cf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321692524Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321719501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"0","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321755416Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"0","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321771373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"1","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321786271Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"1","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321800109Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"2","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321814637Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"2","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321828255Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321843153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.321858011Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"ds_connection connected from Ok(127.0.0.1:59663)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.32194026Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"ds_connection connected from Ok(127.0.0.1:58142)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.322104088Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"ds_connection connected from Ok(127.0.0.1:62284)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.322300941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.32288309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"1","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.322924415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"0","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.323517183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"1","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.323671482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"0","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.324095274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.32426541Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.324405791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"2","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.32491903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"2","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[0]R flush_numbers[0..12]: [132, 153, 141, 147, 148, 152, 152, 152, 142, 147, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325446308Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[0]R generation[0..12]: [74, 85, 80, 82, 83, 84, 84, 84, 80, 82, 83, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325471484Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325487232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[1]R flush_numbers[0..12]: [132, 153, 141, 147, 148, 152, 152, 152, 142, 147, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.32550158Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[1]R generation[0..12]: [74, 85, 80, 82, 83, 84, 84, 84, 80, 82, 83, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325516018Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325531036Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[2]R flush_numbers[0..12]: [132, 153, 141, 147, 148, 152, 152, 152, 142, 147, 148, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325545474Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[2]R generation[0..12]: [74, 85, 80, 82, 83, 84, 84, 84, 80, 82, 83, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325559492Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.3255745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325592008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325612625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325626543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325640081Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"":"downstairs","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325654049Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325666498Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"0","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325680576Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"1","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325694284Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"client":"2","":"downstairs","session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325708122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.32572091Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"a111a42f-f08b-41e2-8179-05d708b815cf is now active with session: 4d7d30f1-f9f7-460e-872d-8c09d4030a20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325734048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325746347Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536,"session_id":"4d7d30f1-f9f7-460e-872d-8c09d4030a20"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325758885Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} {"msg":"Activated sub_volume a111a42f-f08b-41e2-8179-05d708b815cf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:10.325770603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 8192 data: 5 5 02/30 Read block 52 len 16384 03/30 Read block 483 len 20480 04/30 Read block 541 len 4096 05/30 Flush 06/30 Flush 07/30 Write block 303 len 12288 data: 3 4 4 08/30 Flush 09/30 Write block 515 len 40960 data: 12 9 6 5 5 8 8 8 7 7 10/30 Write block 180 len 32768 data: 8 9 8 7 6 7 7 7 11/30 Read block 319 len 24576 12/30 Read block 124 len 8192 13/30 Flush 14/30 Write block 227 len 24576 data: 8 10 8 10 8 11 15/30 Write block 311 len 12288 data: 5 6 6 16/30 Read block 274 len 4096 17/30 Read block 359 len 32768 18/30 Read block 263 len 8192 19/30 Read block 567 len 40960 20/30 Write block 533 len 20480 data: 9 9 8 8 8 21/30 Flush 22/30 Write block 486 len 8192 data: 7 6 23/30 Read block 317 len 12288 24/30 Flush 25/30 Write block 402 len 36864 data: 7 9 8 7 6 7 7 7 7 26/30 Write block 451 len 12288 data: 12 10 9 27/30 Read block 281 len 12288 28/30 Read block 141 len 32768 29/30 Write block 117 len 24576 data: 8 8 12 11 13 11 30/30 Read block 385 len 36864 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:10.346116485Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 83 83 83 148 148 148 F F F 5 100-119 84 84 84 152 152 152 T T T 6 120-139 84 84 84 152 152 152 T T T 7 140-159 84 84 84 152 152 152 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 86 86 86 157 157 157 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 86 86 86 158 158 158 F F F 12 240-259 84 84 84 152 152 152 F F F 13 260-279 81 81 81 143 143 143 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 86 86 86 158 158 158 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 82 82 82 146 146 146 T T T 21 420-439 78 78 78 138 138 138 F F F 22 440-459 85 85 85 153 153 153 T T T 23 460-479 82 82 82 147 147 147 F F F 24 480-499 86 86 86 159 159 159 F F F 25 500-519 86 86 86 157 157 157 F F F 26 520-539 86 86 86 158 158 158 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 84 84 84 150 150 150 F F F 29 580-599 84 84 84 150 150 150 F F F Max gen: 86, Max flush: 159 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 83 83 83 148 148 148 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 87 87 87 160 160 160 F F F 7 140-159 84 84 84 152 152 152 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 86 86 86 157 157 157 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 86 86 86 158 158 158 F F F 12 240-259 84 84 84 152 152 152 F F F 13 260-279 81 81 81 143 143 143 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 86 86 86 158 158 158 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 87 87 87 160 160 160 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 87 87 87 160 160 160 F F F 23 460-479 82 82 82 147 147 147 F F F 24 480-499 86 86 86 159 159 159 F F F 25 500-519 86 86 86 157 157 157 F F F 26 520-539 86 86 86 158 158 158 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 84 84 84 150 150 150 F F F 29 580-599 84 84 84 150 150 150 F F F Max gen: 87, Max flush: 160 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.592776693Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.637932802Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.638881312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.638916407Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.638930315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.638944813Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} {"msg":"Crucible 4c50891c-503e-492a-8163-41d86ebc1ceb has session id: d9558dd0-0383-4efb-960a-3f3c6de010a5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.639528863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Upstairs opts: Upstairs UUID: 4c50891c-503e-492a-8163-41d86ebc1ceb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.639566118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Crucible stats registered with UUID: 4c50891c-503e-492a-8163-41d86ebc1ceb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.639587915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:14.639601533Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640037103Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.64005948Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} {"msg":"4c50891c-503e-492a-8163-41d86ebc1ceb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640126051Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640154637Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"0","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640184733Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"0","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.64020084Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640222317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640248704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"2","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.64027756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"2","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640306546Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640334902Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640360658Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:34972)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640387345Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:14.640452156Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:14.640489201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:14.640522546Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:52827)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640591127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.640626432Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.641135842Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"0","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.641164908Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"2","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.641807379Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"2","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.641836575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"0","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.642440242Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:14.642469628Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.642005348Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53326)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.64213667Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.643200034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.643825848Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[0]R flush_numbers[0..12]: [132, 153, 141, 147, 148, 160, 160, 152, 142, 157, 148, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644485917Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[0]R generation[0..12]: [74, 85, 80, 82, 83, 87, 87, 84, 80, 86, 83, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644521602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644542249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[1]R flush_numbers[0..12]: [132, 153, 141, 147, 148, 160, 160, 152, 142, 157, 148, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644560626Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[1]R generation[0..12]: [74, 85, 80, 82, 83, 87, 87, 84, 80, 86, 83, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644578254Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644596322Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[2]R flush_numbers[0..12]: [132, 153, 141, 147, 148, 160, 160, 152, 142, 157, 148, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644614159Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[2]R generation[0..12]: [74, 85, 80, 82, 83, 87, 87, 84, 80, 86, 83, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644631637Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644649484Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644666042Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.64468305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644700367Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644716805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"":"downstairs","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644734033Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.64475408Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"0","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:24.644771708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"1","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644792625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"client":"2","":"downstairs","session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644810132Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.64482585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"4c50891c-503e-492a-8163-41d86ebc1ceb is now active with session: d9558dd0-0383-4efb-960a-3f3c6de010a5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644842398Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644857746Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544,"session_id":"d9558dd0-0383-4efb-960a-3f3c6de010a5"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644873894Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} {"msg":"Activated sub_volume 4c50891c-503e-492a-8163-41d86ebc1ceb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:24.644885552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 255 len 24576 data: 10 10 10 11 12 11 02/30 Write block 188 len 36864 data: 8 8 9 10 8 7 6 5 6 03/30 Write block 502 len 20480 data: 7 9 9 7 8 04/30 Flush 05/30 Write block 508 len 24576 data: 5 6 8 8 9 9 06/30 Write block 448 len 40960 data: 15 15 13 13 11 10 7 6 5 6 07/30 Read block 216 len 24576 08/30 Read block 51 len 36864 09/30 Read block 412 len 16384 10/30 Write block 330 len 8192 data: 5 6 11/30 Flush 12/30 Write block 589 len 8192 data: 7 10 13/30 Write block 329 len 16384 data: 7 6 7 6 14/30 Read block 364 len 40960 15/30 Flush 16/30 Write block 262 len 24576 data: 10 9 9 8 7 6 17/30 Read block 387 len 40960 18/30 Write block 444 len 32768 data: 9 10 11 14 16 16 14 14 19/30 Read block 555 len 8192 20/30 Read block 504 len 36864 21/30 Write block 592 len 28672 data: 8 8 9 7 6 6 5 22/30 Write block 124 len 24576 data: 11 12 12 8 8 8 23/30 Read block 339 len 4096 24/30 Write block 225 len 28672 data: 9 9 9 11 9 11 9 25/30 Write block 91 len 24576 data: 12 13 13 11 9 10 26/30 Read block 33 len 8192 27/30 Read block 323 len 16384 28/30 Write block 302 len 4096 data: 3 29/30 Write block 86 len 28672 data: 11 10 11 10 12 13 14 30/30 Read block 371 len 36864 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 13 15 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:24.665990919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 83 83 83 148 148 148 T T T 5 100-119 87 87 87 160 160 160 F F F 6 120-139 87 87 87 160 160 160 T T T 7 140-159 84 84 84 152 152 152 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 88 88 88 161 161 161 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 86 86 86 158 158 158 T T T 12 240-259 88 88 88 161 161 161 F F F 13 260-279 88 88 88 161 161 161 T T T 14 280-299 82 82 82 146 146 146 F F F 15 300-319 86 86 86 158 158 158 T T T 16 320-339 88 88 88 163 163 163 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 87 87 87 160 160 160 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 88 88 88 162 162 162 T T T 23 460-479 82 82 82 147 147 147 F F F 24 480-499 86 86 86 159 159 159 F F F 25 500-519 88 88 88 162 162 162 F F F 26 520-539 86 86 86 158 158 158 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 84 84 84 150 150 150 F F F 29 580-599 88 88 88 163 163 163 T T T Max gen: 88, Max flush: 163 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 89 89 89 164 164 164 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 89 89 89 164 164 164 F F F 7 140-159 84 84 84 152 152 152 F F F 8 160-179 80 80 80 142 142 142 F F F 9 180-199 88 88 88 161 161 161 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 89 89 89 164 164 164 F F F 12 240-259 88 88 88 161 161 161 F F F 13 260-279 89 89 89 164 164 164 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 89 89 89 164 164 164 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 87 87 87 160 160 160 F F F 21 420-439 78 78 78 138 138 138 F F F 22 440-459 89 89 89 164 164 164 F F F 23 460-479 82 82 82 147 147 147 F F F 24 480-499 86 86 86 159 159 159 F F F 25 500-519 88 88 88 162 162 162 F F F 26 520-539 86 86 86 158 158 158 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 84 84 84 150 150 150 F F F 29 580-599 89 89 89 164 164 164 F F F Max gen: 89, Max flush: 164 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.925889735Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.970650889Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.9714437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.971480415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.971495223Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.971510181Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} {"msg":"Crucible 28f5e849-9b05-491a-adbb-e10cbc4d73f0 has session id: 96641603-d304-4a90-a05e-01710ed49412","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972016831Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Upstairs opts: Upstairs UUID: 28f5e849-9b05-491a-adbb-e10cbc4d73f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972062395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Crucible stats registered with UUID: 28f5e849-9b05-491a-adbb-e10cbc4d73f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972089141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:28.972111878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972519942Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972551338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} {"msg":"28f5e849-9b05-491a-adbb-e10cbc4d73f0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.97260307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972630187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"0","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972668721Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"0","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972685229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972700517Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972714315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"2","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972729403Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"2","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.97275207Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972774007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.972792934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"ds_connection connected from Ok(127.0.0.1:56981)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.973013494Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"ds_connection connected from Ok(127.0.0.1:35754)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.973054498Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:28.973081674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:28.973110621Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:28.973207217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.973275618Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.973789377Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"2","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.973818823Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"0","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.974484552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"2","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.974514037Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"0","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.975107536Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:28.975130043Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.975520495Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"ds_connection connected from Ok(127.0.0.1:44726)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.975662835Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.976694363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977239979Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[0]R flush_numbers[0..12]: [132, 153, 141, 147, 164, 160, 164, 152, 142, 161, 148, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.97780866Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[0]R generation[0..12]: [74, 85, 80, 82, 89, 87, 89, 84, 80, 88, 83, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977839236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977855834Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[1]R flush_numbers[0..12]: [132, 153, 141, 147, 164, 160, 164, 152, 142, 161, 148, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977870572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[1]R generation[0..12]: [74, 85, 80, 82, 89, 87, 89, 84, 80, 88, 83, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.97788502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977899418Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[2]R flush_numbers[0..12]: [132, 153, 141, 147, 164, 160, 164, 152, 142, 161, 148, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977913576Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[2]R generation[0..12]: [74, 85, 80, 82, 89, 87, 89, 84, 80, 88, 83, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977927404Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977941572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.97795479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977968378Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.977981886Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978010793Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"":"downstairs","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.97802632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"Wait for a query_work_queue command to finish before sending IO 96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978044108Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"0","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978058076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"1","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978073964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"client":"2","":"downstairs","session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978087652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.97810065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"28f5e849-9b05-491a-adbb-e10cbc4d73f0 is now active with session: 96641603-d304-4a90-a05e-01710ed49412","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978113878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978126226Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552,"session_id":"96641603-d304-4a90-a05e-01710ed49412"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978138855Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} {"msg":"Activated sub_volume 28f5e849-9b05-491a-adbb-e10cbc4d73f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:38.978150553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 455 len 28672 data: 7 6 7 7 6 4 4 02/30 Flush 03/30 Write block 173 len 4096 data: 9 04/30 Write block 511 len 32768 data: 9 10 10 12 13 10 7 6 05/30 Read block 238 len 28672 06/30 Read block 291 len 28672 07/30 Flush 08/30 Write block 425 len 16384 data: 9 8 7 6 09/30 Write block 187 len 12288 data: 8 9 9 10/30 Write block 133 len 40960 data: 7 6 8 5 5 5 4 5 5 6 11/30 Write block 550 len 28672 data: 8 9 10 9 9 9 8 12/30 Write block 154 len 20480 data: 10 11 13 12 10 13/30 Read block 487 len 16384 14/30 Flush 15/30 Write block 588 len 28672 data: 8 8 11 8 9 9 10 16/30 Read block 451 len 20480 17/30 Read block 229 len 28672 18/30 Write block 513 len 8192 data: 11 13 19/30 Read block 511 len 36864 20/30 Read block 390 len 32768 21/30 Read block 275 len 4096 22/30 Read block 361 len 4096 23/30 Write block 575 len 20480 data: 4 4 5 8 8 24/30 Write block 526 len 36864 data: 11 8 9 10 10 10 10 10 10 25/30 Write block 577 len 20480 data: 6 9 9 8 7 26/30 Read block 407 len 28672 27/30 Read block 186 len 20480 28/30 Read block 162 len 24576 29/30 Read block 199 len 32768 30/30 Write block 167 len 32768 data: 6 7 10 10 10 10 10 7 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:38.999198748Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 89 89 89 164 164 164 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 90 90 90 167 167 167 F F F 7 140-159 90 90 90 167 167 167 F F F 8 160-179 90 90 90 166 166 166 T T T 9 180-199 90 90 90 167 167 167 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 89 89 89 164 164 164 F F F 12 240-259 88 88 88 161 161 161 F F F 13 260-279 89 89 89 164 164 164 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 89 89 89 164 164 164 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 87 87 87 160 160 160 F F F 21 420-439 90 90 90 167 167 167 F F F 22 440-459 90 90 90 165 165 165 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 86 86 86 159 159 159 F F F 25 500-519 90 90 90 166 166 166 T T T 26 520-539 86 86 86 158 158 158 T T T 27 540-559 90 90 90 167 167 167 F F F 28 560-579 84 84 84 150 150 150 T T T 29 580-599 89 89 89 164 164 164 T T T Max gen: 90, Max flush: 167 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 F F F 2 040-059 80 80 80 141 141 141 F F F 3 060-079 82 82 82 147 147 147 F F F 4 080-099 89 89 89 164 164 164 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 90 90 90 167 167 167 F F F 7 140-159 90 90 90 167 167 167 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 90 90 90 167 167 167 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 89 89 89 164 164 164 F F F 12 240-259 88 88 88 161 161 161 F F F 13 260-279 89 89 89 164 164 164 F F F 14 280-299 82 82 82 146 146 146 F F F 15 300-319 89 89 89 164 164 164 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 82 82 82 145 145 145 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 79 79 79 140 140 140 F F F 20 400-419 87 87 87 160 160 160 F F F 21 420-439 90 90 90 167 167 167 F F F 22 440-459 90 90 90 165 165 165 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 86 86 86 159 159 159 F F F 25 500-519 91 91 91 168 168 168 F F F 26 520-539 91 91 91 168 168 168 F F F 27 540-559 90 90 90 167 167 167 F F F 28 560-579 91 91 91 168 168 168 F F F 29 580-599 91 91 91 168 168 168 F F F Max gen: 91, Max flush: 168 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.248610782Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:53.291412575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.291833037Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.323281502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.32402361Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.324061024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.324075023Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.324089701Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} {"msg":"Crucible 1363d743-67c4-4e33-b465-3ea1060c0f67 has session id: 308de055-85f6-405e-ad11-eb18ebbaa33a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.324651323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Upstairs opts: Upstairs UUID: 1363d743-67c4-4e33-b465-3ea1060c0f67, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.324697397Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Crucible stats registered with UUID: 1363d743-67c4-4e33-b465-3ea1060c0f67","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.324713565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:53.324727843Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325250891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325276407Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} {"msg":"1363d743-67c4-4e33-b465-3ea1060c0f67 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325343538Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325362365Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"0","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325391652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"0","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.32540633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"1","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325422027Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"1","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325435975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"2","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325458922Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"2","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.32547948Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325502826Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325526163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40309)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.32569339Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46556)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325803805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46618)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.325835811Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.326321764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"1","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.326375146Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"2","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.326405102Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"0","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327021327Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"1","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327053103Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"2","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327069551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"0","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327618695Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327723081Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[0]R flush_numbers[0..12]: [132, 153, 141, 147, 164, 160, 167, 167, 168, 167, 148, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327789602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[0]R generation[0..12]: [74, 85, 80, 82, 89, 87, 90, 90, 91, 90, 83, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327809529Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327824597Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[1]R flush_numbers[0..12]: [132, 153, 141, 147, 164, 160, 167, 167, 168, 167, 148, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327839075Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[1]R generation[0..12]: [74, 85, 80, 82, 89, 87, 90, 90, 91, 90, 83, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327853373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327868191Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[2]R flush_numbers[0..12]: [132, 153, 141, 147, 164, 160, 167, 167, 168, 167, 148, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327885558Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[2]R generation[0..12]: [74, 85, 80, 82, 89, 87, 90, 90, 91, 90, 83, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327900067Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327914385Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327928033Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327941661Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327955069Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327975506Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"":"downstairs","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.327989274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328001613Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"0","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328015351Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"1","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328029069Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"client":"2","":"downstairs","session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328042787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328055415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"1363d743-67c4-4e33-b465-3ea1060c0f67 is now active with session: 308de055-85f6-405e-ad11-eb18ebbaa33a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328068623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328081022Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560,"session_id":"308de055-85f6-405e-ad11-eb18ebbaa33a"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.32809534Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} {"msg":"Activated sub_volume 1363d743-67c4-4e33-b465-3ea1060c0f67","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:53.328107078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 397 len 32768 data: 7 6 5 7 7 8 10 9 02/30 Write block 168 len 40960 data: 8 11 11 11 11 11 8 7 6 6 03/30 Write block 295 len 20480 data: 5 5 5 5 5 04/30 Read block 318 len 28672 05/30 Read block 440 len 24576 06/30 Read block 537 len 40960 07/30 Read block 209 len 8192 08/30 Read block 170 len 24576 09/30 Read block 254 len 40960 10/30 Read block 134 len 12288 11/30 Read block 4 len 36864 12/30 Flush 13/30 Write block 567 len 4096 data: 7 14/30 Write block 226 len 16384 data: 10 10 12 10 15/30 Read block 436 len 4096 16/30 Read block 83 len 32768 17/30 Write block 454 len 24576 data: 8 8 7 8 8 7 18/30 Read block 361 len 12288 19/30 Read block 239 len 8192 20/30 Write block 426 len 40960 data: 9 8 7 7 7 5 3 3 3 2 21/30 Read block 597 len 12288 22/30 Read block 229 len 24576 23/30 Write block 433 len 16384 data: 4 4 3 2 24/30 Read block 109 len 16384 25/30 Write block 539 len 28672 data: 9 7 5 5 5 4 6 26/30 Write block 70 len 8192 data: 6 6 27/30 Write block 481 len 32768 data: 6 6 8 11 10 8 7 5 28/30 Write block 36 len 36864 data: 4 3 4 8 10 12 14 14 14 29/30 Write block 342 len 32768 data: 9 9 12 11 9 9 9 8 30/30 Read block 293 len 40960 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 1 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:53.350306065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 85 85 85 153 153 153 T T T 2 040-059 80 80 80 141 141 141 T T T 3 060-079 82 82 82 147 147 147 T T T 4 080-099 89 89 89 164 164 164 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 90 90 90 167 167 167 F F F 7 140-159 90 90 90 167 167 167 F F F 8 160-179 92 92 92 169 169 169 F F F 9 180-199 90 90 90 167 167 167 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 89 89 89 164 164 164 T T T 12 240-259 88 88 88 161 161 161 F F F 13 260-279 89 89 89 164 164 164 F F F 14 280-299 92 92 92 169 169 169 F F F 15 300-319 89 89 89 164 164 164 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 82 82 82 145 145 145 T T T 18 360-379 82 82 82 147 147 147 F F F 19 380-399 92 92 92 169 169 169 F F F 20 400-419 92 92 92 169 169 169 F F F 21 420-439 90 90 90 167 167 167 T T T 22 440-459 90 90 90 165 165 165 T T T 23 460-479 90 90 90 165 165 165 F F F 24 480-499 86 86 86 159 159 159 T T T 25 500-519 91 91 91 168 168 168 F F F 26 520-539 91 91 91 168 168 168 T T T 27 540-559 90 90 90 167 167 167 T T T 28 560-579 91 91 91 168 168 168 T T T 29 580-599 91 91 91 168 168 168 F F F Max gen: 92, Max flush: 169 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 93 93 93 170 170 170 F F F 2 040-059 93 93 93 170 170 170 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 89 89 89 164 164 164 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 90 90 90 167 167 167 F F F 7 140-159 90 90 90 167 167 167 F F F 8 160-179 92 92 92 169 169 169 F F F 9 180-199 90 90 90 167 167 167 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 93 93 93 170 170 170 F F F 12 240-259 88 88 88 161 161 161 F F F 13 260-279 89 89 89 164 164 164 F F F 14 280-299 92 92 92 169 169 169 F F F 15 300-319 89 89 89 164 164 164 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 93 93 93 170 170 170 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 92 92 92 169 169 169 F F F 20 400-419 92 92 92 169 169 169 F F F 21 420-439 93 93 93 170 170 170 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 93 93 93 170 170 170 F F F 25 500-519 91 91 91 168 168 168 F F F 26 520-539 93 93 93 170 170 170 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 93 93 93 170 170 170 F F F 29 580-599 91 91 91 168 168 168 F F F Max gen: 93, Max flush: 170 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.658156657Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.703638261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.704359762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.704400517Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.704415244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.704429873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} {"msg":"Crucible e11059c2-2634-4c9a-b073-729eda3a3b59 has session id: 154b7c3f-c769-4553-8894-bf274b8e168a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.704951331Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Upstairs opts: Upstairs UUID: e11059c2-2634-4c9a-b073-729eda3a3b59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.704994975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Crucible stats registered with UUID: e11059c2-2634-4c9a-b073-729eda3a3b59","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705014252Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:57.70503069Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705544319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705573225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} {"msg":"e11059c2-2634-4c9a-b073-729eda3a3b59 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705657164Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705674711Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"0","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705702318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"0","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705725514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"1","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.70575335Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"1","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705784106Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"2","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705813542Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"2","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705838029Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705865435Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705892211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57751)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.705995117Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"ds_connection connected from Ok(127.0.0.1:35882)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.706028902Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40556)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.706147886Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.706781249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"2","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.706856059Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"0","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.70713495Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"1","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.707562102Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"2","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.707593107Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"0","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.707696953Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"1","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708177337Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708290691Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[0]R flush_numbers[0..12]: [132, 170, 170, 170, 164, 160, 167, 167, 169, 167, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[0]R generation[0..12]: [74, 93, 93, 93, 89, 87, 90, 90, 92, 90, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708396907Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708412395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[1]R flush_numbers[0..12]: [132, 170, 170, 170, 164, 160, 167, 167, 169, 167, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708427083Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[1]R generation[0..12]: [74, 93, 93, 93, 89, 87, 90, 90, 92, 90, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.70844307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708458288Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[2]R flush_numbers[0..12]: [132, 170, 170, 170, 164, 160, 167, 167, 169, 167, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708472866Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[2]R generation[0..12]: [74, 93, 93, 93, 89, 87, 90, 90, 92, 90, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708487074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708501333Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708514671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708528309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708541747Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708565224Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"":"downstairs","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708579482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.70859196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"0","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708605768Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"1","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708619336Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"client":"2","":"downstairs","session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708632894Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708645443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"e11059c2-2634-4c9a-b073-729eda3a3b59 is now active with session: 154b7c3f-c769-4553-8894-bf274b8e168a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708658491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708670979Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568,"session_id":"154b7c3f-c769-4553-8894-bf274b8e168a"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708683557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} {"msg":"Activated sub_volume e11059c2-2634-4c9a-b073-729eda3a3b59","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:57.708695026Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 486 len 28672 data: 9 8 6 6 7 6 7 02/30 Read block 426 len 20480 03/30 Read block 483 len 4096 04/30 Write block 513 len 40960 data: 12 14 14 11 8 7 6 9 9 9 05/30 Read block 385 len 4096 06/30 Write block 275 len 12288 data: 5 4 4 07/30 Write block 168 len 32768 data: 9 12 12 12 12 12 9 8 08/30 Write block 131 len 40960 data: 7 6 8 7 9 6 6 6 5 6 09/30 Read block 286 len 24576 10/30 Flush 11/30 Write block 515 len 28672 data: 15 12 9 8 7 10 10 12/30 Read block 456 len 24576 13/30 Read block 435 len 4096 14/30 Flush 15/30 Write block 483 len 36864 data: 9 12 11 10 9 7 7 8 7 16/30 Write block 259 len 16384 data: 13 12 9 11 17/30 Write block 407 len 12288 data: 8 8 8 18/30 Write block 154 len 16384 data: 11 12 14 13 19/30 Write block 579 len 8192 data: 10 9 20/30 Read block 135 len 36864 21/30 Write block 338 len 20480 data: 10 9 9 7 10 22/30 Read block 143 len 12288 23/30 Write block 152 len 24576 data: 9 9 12 13 15 14 24/30 Read block 91 len 4096 25/30 Read block 16 len 12288 26/30 Read block 73 len 4096 27/30 Write block 260 len 40960 data: 13 10 12 10 10 9 8 7 6 3 28/30 Read block 481 len 8192 29/30 Write block 132 len 24576 data: 7 9 8 10 7 7 30/30 Read block 486 len 28672 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 9 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 15 16 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:57.729492845Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 93 93 93 170 170 170 F F F 2 040-059 93 93 93 170 170 170 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 89 89 89 164 164 164 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 94 94 94 171 171 171 T T T 7 140-159 94 94 94 171 171 171 T T T 8 160-179 94 94 94 171 171 171 F F F 9 180-199 90 90 90 167 167 167 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 93 93 93 170 170 170 F F F 12 240-259 88 88 88 161 161 161 T T T 13 260-279 94 94 94 171 171 171 T T T 14 280-299 92 92 92 169 169 169 F F F 15 300-319 89 89 89 164 164 164 F F F 16 320-339 88 88 88 163 163 163 T T T 17 340-359 93 93 93 170 170 170 T T T 18 360-379 82 82 82 147 147 147 F F F 19 380-399 92 92 92 169 169 169 F F F 20 400-419 92 92 92 169 169 169 T T T 21 420-439 93 93 93 170 170 170 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 94 94 94 171 171 171 T T T 25 500-519 94 94 94 172 172 172 F F F 26 520-539 94 94 94 172 172 172 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 93 93 93 170 170 170 T T T 29 580-599 91 91 91 168 168 168 T T T Max gen: 94, Max flush: 172 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 F F F 1 020-039 93 93 93 170 170 170 F F F 2 040-059 93 93 93 170 170 170 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 89 89 89 164 164 164 F F F 5 100-119 87 87 87 160 160 160 F F F 6 120-139 95 95 95 173 173 173 F F F 7 140-159 95 95 95 173 173 173 F F F 8 160-179 94 94 94 171 171 171 F F F 9 180-199 90 90 90 167 167 167 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 93 93 93 170 170 170 F F F 12 240-259 95 95 95 173 173 173 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 92 92 92 169 169 169 F F F 15 300-319 89 89 89 164 164 164 F F F 16 320-339 95 95 95 173 173 173 F F F 17 340-359 95 95 95 173 173 173 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 92 92 92 169 169 169 F F F 20 400-419 95 95 95 173 173 173 F F F 21 420-439 93 93 93 170 170 170 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 95 95 95 173 173 173 F F F 25 500-519 94 94 94 172 172 172 F F F 26 520-539 94 94 94 172 172 172 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 95 95 95 173 173 173 F F F 29 580-599 95 95 95 173 173 173 F F F Max gen: 95, Max flush: 173 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.003646808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.04936029Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.050313259Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.050352094Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.050366732Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.05038138Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} {"msg":"Crucible 18e6ecdf-f798-45c2-9663-cd052155bb9d has session id: 370f6958-90f7-41a7-a5b9-281aeaf5ae94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.050932374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Upstairs opts: Upstairs UUID: 18e6ecdf-f798-45c2-9663-cd052155bb9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.05103525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Crucible stats registered with UUID: 18e6ecdf-f798-45c2-9663-cd052155bb9d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051074584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:12.051098401Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051754051Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051789046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} {"msg":"18e6ecdf-f798-45c2-9663-cd052155bb9d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051842609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051869305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"0","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051900851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"0","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051916928Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"1","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051931846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"1","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051946144Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"2","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051961102Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"2","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051974591Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.051993638Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.052018965Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"ds_connection connected from Ok(127.0.0.1:40695)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.052045641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"ds_connection connected from Ok(127.0.0.1:61130)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.052073487Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"ds_connection connected from Ok(127.0.0.1:51508)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.052310574Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.052962585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"0","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.052991571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"1","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.053102416Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"2","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.053644251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"1","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.053672827Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"0","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.053696364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"2","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054254617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054279414Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[0]R flush_numbers[0..12]: [132, 170, 170, 170, 164, 160, 173, 173, 171, 167, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054357413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[0]R generation[0..12]: [74, 93, 93, 93, 89, 87, 95, 95, 94, 90, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.05437843Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054394078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[1]R flush_numbers[0..12]: [132, 170, 170, 170, 164, 160, 173, 173, 171, 167, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054408896Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[1]R generation[0..12]: [74, 93, 93, 93, 89, 87, 95, 95, 94, 90, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054423254Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054441701Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[2]R flush_numbers[0..12]: [132, 170, 170, 170, 164, 160, 173, 173, 171, 167, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054456539Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[2]R generation[0..12]: [74, 93, 93, 93, 89, 87, 95, 95, 94, 90, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054470547Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054484935Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054498404Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054511952Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.0545253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054538998Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"":"downstairs","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054552726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054565094Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"0","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054586292Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"1","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054599839Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"client":"2","":"downstairs","session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054613558Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054626086Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"18e6ecdf-f798-45c2-9663-cd052155bb9d is now active with session: 370f6958-90f7-41a7-a5b9-281aeaf5ae94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054639174Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054653292Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576,"session_id":"370f6958-90f7-41a7-a5b9-281aeaf5ae94"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.05466615Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} {"msg":"Activated sub_volume 18e6ecdf-f798-45c2-9663-cd052155bb9d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:12.054677619Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 330 len 8192 data: 7 8 02/30 Read block 579 len 12288 03/30 Write block 135 len 24576 data: 11 8 8 7 6 7 04/30 Write block 594 len 24576 data: 11 8 7 7 6 4 05/30 Read block 89 len 28672 06/30 Read block 191 len 32768 07/30 Write block 167 len 32768 data: 7 10 13 13 13 13 13 10 08/30 Flush 09/30 Write block 117 len 28672 data: 9 9 13 12 14 12 9 10/30 Write block 295 len 12288 data: 6 6 6 11/30 Flush 12/30 Write block 403 len 32768 data: 11 10 8 7 9 9 9 8 13/30 Read block 538 len 40960 14/30 Write block 149 len 8192 data: 10 10 15/30 Read block 562 len 20480 16/30 Write block 33 len 40960 data: 5 3 3 5 4 5 9 11 13 15 17/30 Write block 280 len 40960 data: 8 7 7 6 5 5 6 7 6 6 18/30 Write block 50 len 36864 data: 5 7 6 8 8 11 10 10 10 19/30 Read block 496 len 36864 20/30 Flush 21/30 Write block 180 len 16384 data: 9 10 9 8 22/30 Read block 100 len 24576 23/30 Write block 347 len 32768 data: 10 10 9 8 9 10 10 10 24/30 Write block 89 len 36864 data: 11 13 14 15 14 12 10 11 10 25/30 Read block 262 len 36864 26/30 Write block 308 len 40960 data: 4 3 3 6 7 7 8 7 7 5 27/30 Write block 244 len 32768 data: 10 11 10 11 13 10 9 5 28/30 Write block 19 len 4096 data: 11 29/30 Read block 520 len 8192 30/30 Read block 213 len 40960 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:12.076276708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 132 132 132 T T T 1 020-039 96 96 96 176 176 176 F F F 2 040-059 96 96 96 176 176 176 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 89 89 89 164 164 164 T T T 5 100-119 96 96 96 175 175 175 F F F 6 120-139 96 96 96 175 175 175 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 96 96 96 174 174 174 F F F 9 180-199 90 90 90 167 167 167 T T T 10 200-219 83 83 83 148 148 148 F F F 11 220-239 93 93 93 170 170 170 F F F 12 240-259 95 95 95 173 173 173 T T T 13 260-279 95 95 95 173 173 173 F F F 14 280-299 96 96 96 176 176 176 F F F 15 300-319 89 89 89 164 164 164 T T T 16 320-339 96 96 96 174 174 174 F F F 17 340-359 95 95 95 173 173 173 T T T 18 360-379 82 82 82 147 147 147 F F F 19 380-399 92 92 92 169 169 169 F F F 20 400-419 96 96 96 176 176 176 F F F 21 420-439 93 93 93 170 170 170 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 95 95 95 173 173 173 F F F 25 500-519 94 94 94 172 172 172 F F F 26 520-539 94 94 94 172 172 172 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 95 95 95 173 173 173 F F F 29 580-599 96 96 96 174 174 174 F F F Max gen: 96, Max flush: 176 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 F F F 2 040-059 96 96 96 176 176 176 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 97 97 97 177 177 177 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 96 96 96 175 175 175 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 96 96 96 174 174 174 F F F 9 180-199 97 97 97 177 177 177 F F F 10 200-219 83 83 83 148 148 148 F F F 11 220-239 93 93 93 170 170 170 F F F 12 240-259 97 97 97 177 177 177 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 96 96 96 176 176 176 F F F 15 300-319 97 97 97 177 177 177 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 97 97 97 177 177 177 F F F 18 360-379 82 82 82 147 147 147 F F F 19 380-399 92 92 92 169 169 169 F F F 20 400-419 96 96 96 176 176 176 F F F 21 420-439 93 93 93 170 170 170 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 95 95 95 173 173 173 F F F 25 500-519 94 94 94 172 172 172 F F F 26 520-539 94 94 94 172 172 172 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 95 95 95 173 173 173 F F F 29 580-599 96 96 96 174 174 174 F F F Max gen: 97, Max flush: 177 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.32912813Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:26.372128876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.372626757Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.404423214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.405217085Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.405260239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.405277587Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.405295804Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} {"msg":"Crucible f91419a3-d4f4-4498-9b08-a30d243970a6 has session id: 6fb1890f-0dbe-477a-ac64-065d432b6b52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.405873605Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Upstairs opts: Upstairs UUID: f91419a3-d4f4-4498-9b08-a30d243970a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.405920258Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Crucible stats registered with UUID: f91419a3-d4f4-4498-9b08-a30d243970a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.405940555Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:26.405957783Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406329252Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406362037Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} {"msg":"f91419a3-d4f4-4498-9b08-a30d243970a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.4064195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406450905Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"0","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.40649186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"0","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406530354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"1","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.40656289Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"1","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406590086Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"2","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406619992Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"2","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406650258Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406684203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406712879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"ds_connection connected from Ok(127.0.0.1:55489)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406762053Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"ds_connection connected from Ok(127.0.0.1:60004)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406798057Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"ds_connection connected from Ok(127.0.0.1:35500)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.406829393Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.407385897Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"1","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.407566622Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"2","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.407611406Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"0","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408043956Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"1","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408079041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"2","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.40837766Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"0","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408646983Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.40873945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[0]R flush_numbers[0..12]: [177, 176, 176, 170, 177, 175, 175, 176, 174, 177, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408894039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[0]R generation[0..12]: [97, 96, 96, 93, 97, 96, 96, 96, 96, 97, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408915226Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408930554Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[1]R flush_numbers[0..12]: [177, 176, 176, 170, 177, 175, 175, 176, 174, 177, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408944922Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[1]R generation[0..12]: [97, 96, 96, 93, 97, 96, 96, 96, 96, 97, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.40895929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408973678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[2]R flush_numbers[0..12]: [177, 176, 176, 170, 177, 175, 175, 176, 174, 177, 148, 170]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.408987946Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[2]R generation[0..12]: [97, 96, 96, 93, 97, 96, 96, 96, 96, 97, 83, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409001974Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409016552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409029971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409046728Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409060546Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:07:26.409074024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"":"downstairs","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409094952Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.4091074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"0","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409121198Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"1","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409135106Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"client":"2","":"downstairs","session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409149014Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409161603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"f91419a3-d4f4-4498-9b08-a30d243970a6 is now active with session: 6fb1890f-0dbe-477a-ac64-065d432b6b52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409174751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409187079Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584,"session_id":"6fb1890f-0dbe-477a-ac64-065d432b6b52"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409199657Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} {"msg":"Activated sub_volume f91419a3-d4f4-4498-9b08-a30d243970a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:26.409211276Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 429 len 36864 data: 8 8 6 4 5 5 4 3 2 02/30 Read block 141 len 16384 03/30 Read block 195 len 8192 04/30 Flush 05/30 Write block 246 len 4096 data: 11 06/30 Read block 201 len 32768 07/30 Write block 503 len 36864 data: 10 10 8 9 6 6 7 9 10 08/30 Read block 403 len 4096 09/30 Write block 528 len 28672 data: 10 11 11 11 11 11 11 10/30 Read block 191 len 36864 11/30 Read block 515 len 12288 12/30 Read block 282 len 8192 13/30 Read block 188 len 8192 14/30 Write block 287 len 16384 data: 8 7 7 7 15/30 Write block 238 len 24576 data: 8 10 11 11 9 10 16/30 Flush 17/30 Write block 422 len 32768 data: 9 8 9 10 10 9 8 9 18/30 Write block 211 len 32768 data: 6 7 10 10 10 8 11 12 19/30 Write block 379 len 24576 data: 4 6 6 7 6 4 20/30 Read block 30 len 20480 21/30 Flush 22/30 Flush 23/30 Write block 588 len 8192 data: 9 9 24/30 Write block 191 len 32768 data: 11 9 8 7 6 7 5 7 25/30 Read block 115 len 20480 26/30 Read block 542 len 36864 27/30 Write block 218 len 4096 data: 13 28/30 Write block 488 len 8192 data: 8 8 29/30 Read block 1 len 20480 30/30 Write block 568 len 8192 data: 7 6 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:26.429150213Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 F F F 2 040-059 96 96 96 176 176 176 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 97 97 97 177 177 177 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 96 96 96 175 175 175 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 96 96 96 174 174 174 F F F 9 180-199 97 97 97 177 177 177 T T T 10 200-219 98 98 98 180 180 180 T T T 11 220-239 98 98 98 179 179 179 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 98 98 98 179 179 179 F F F 15 300-319 97 97 97 177 177 177 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 97 97 97 177 177 177 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 98 98 98 180 180 180 F F F 20 400-419 96 96 96 176 176 176 F F F 21 420-439 98 98 98 180 180 180 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 95 95 95 173 173 173 T T T 25 500-519 98 98 98 179 179 179 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 95 95 95 173 173 173 T T T 29 580-599 96 96 96 174 174 174 T T T Max gen: 98, Max flush: 180 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 F F F 2 040-059 96 96 96 176 176 176 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 97 97 97 177 177 177 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 96 96 96 175 175 175 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 96 96 96 174 174 174 F F F 9 180-199 99 99 99 181 181 181 F F F 10 200-219 99 99 99 181 181 181 F F F 11 220-239 98 98 98 179 179 179 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 98 98 98 179 179 179 F F F 15 300-319 97 97 97 177 177 177 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 97 97 97 177 177 177 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 98 98 98 180 180 180 F F F 20 400-419 96 96 96 176 176 176 F F F 21 420-439 98 98 98 180 180 180 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 99 99 99 181 181 181 F F F 25 500-519 98 98 98 179 179 179 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 99 99 99 181 181 181 F F F 29 580-599 99 99 99 181 181 181 F F F Max gen: 99, Max flush: 181 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.704372493Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.749443294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.75027265Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.750309035Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.750323293Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.750338571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} {"msg":"Crucible 47d4dc56-4dc0-4b1f-9b26-a58d158ba3e4 has session id: 2d1a5f03-f44c-4aed-8882-771a3eeef779","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.750963005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Upstairs opts: Upstairs UUID: 47d4dc56-4dc0-4b1f-9b26-a58d158ba3e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751009668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Crucible stats registered with UUID: 47d4dc56-4dc0-4b1f-9b26-a58d158ba3e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751037435Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:30.751057492Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751519129Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751553644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} {"msg":"47d4dc56-4dc0-4b1f-9b26-a58d158ba3e4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751605027Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751630273Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"0","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751660469Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"0","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751675537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"1","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751690535Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"1","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751704113Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"2","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.75172693Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"2","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751742028Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751756756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751777903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"ds_connection connected from Ok(127.0.0.1:44998)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.751838385Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"ds_connection connected from Ok(127.0.0.1:33491)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.752024509Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"ds_connection connected from Ok(127.0.0.1:61865)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.752070783Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.752556576Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"0","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.752585872Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"1","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.753183389Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"1","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.753214405Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"0","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.753841329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.753969061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.754496449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"2","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.754983692Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"2","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[0]R flush_numbers[0..12]: [177, 176, 176, 170, 177, 175, 175, 176, 174, 181, 181, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755482753Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[0]R generation[0..12]: [97, 96, 96, 93, 97, 96, 96, 96, 96, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.75550706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755526837Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[1]R flush_numbers[0..12]: [177, 176, 176, 170, 177, 175, 175, 176, 174, 181, 181, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755544905Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[1]R generation[0..12]: [97, 96, 96, 93, 97, 96, 96, 96, 96, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755562312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755583399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[2]R flush_numbers[0..12]: [177, 176, 176, 170, 177, 175, 175, 176, 174, 181, 181, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755601817Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[2]R generation[0..12]: [97, 96, 96, 93, 97, 96, 96, 96, 96, 99, 99, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755619454Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755637382Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.75565397Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755670938Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755687765Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755704503Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"":"downstairs","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.75572214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755737568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"0","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755754676Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"1","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:07:30.755771613Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"client":"2","":"downstairs","session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.75579817Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755813858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"47d4dc56-4dc0-4b1f-9b26-a58d158ba3e4 is now active with session: 2d1a5f03-f44c-4aed-8882-771a3eeef779","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755830216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755847643Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592,"session_id":"2d1a5f03-f44c-4aed-8882-771a3eeef779"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755863431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} {"msg":"Activated sub_volume 47d4dc56-4dc0-4b1f-9b26-a58d158ba3e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:30.755878039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 4096 data: 10 02/30 Write block 226 len 36864 data: 11 11 13 11 12 10 12 10 8 03/30 Read block 229 len 40960 04/30 Read block 125 len 28672 05/30 Flush 06/30 Write block 495 len 24576 data: 10 9 9 9 10 9 07/30 Write block 348 len 32768 data: 11 10 9 10 11 11 11 10 08/30 Read block 191 len 20480 09/30 Read block 47 len 32768 10/30 Read block 474 len 40960 11/30 Read block 340 len 28672 12/30 Write block 48 len 28672 data: 8 5 6 8 7 9 9 13/30 Read block 290 len 32768 14/30 Write block 76 len 36864 data: 9 10 10 9 9 11 7 7 9 15/30 Write block 94 len 12288 data: 13 11 12 16/30 Read block 341 len 32768 17/30 Read block 253 len 32768 18/30 Write block 589 len 28672 data: 10 12 9 10 10 12 9 19/30 Read block 362 len 12288 20/30 Write block 55 len 12288 data: 12 11 11 21/30 Write block 568 len 32768 data: 8 7 5 5 5 4 4 5 22/30 Flush 23/30 Write block 42 len 28672 data: 16 15 15 12 11 11 9 24/30 Read block 208 len 36864 25/30 Write block 114 len 24576 data: 4 7 9 10 10 14 26/30 Write block 592 len 20480 data: 11 11 13 10 8 27/30 Write block 381 len 4096 data: 7 28/30 Read block 146 len 16384 29/30 Read block 55 len 24576 30/30 Read block 163 len 12288 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:30.77862332Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 F F F 2 040-059 100 100 100 183 183 183 T T T 3 060-079 100 100 100 183 183 183 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 96 96 96 175 175 175 T T T 6 120-139 96 96 96 175 175 175 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 96 96 96 174 174 174 F F F 9 180-199 99 99 99 181 181 181 F F F 10 200-219 99 99 99 181 181 181 F F F 11 220-239 100 100 100 182 182 182 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 98 98 98 179 179 179 F F F 15 300-319 97 97 97 177 177 177 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 100 100 100 183 183 183 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 98 98 98 180 180 180 T T T 20 400-419 96 96 96 176 176 176 F F F 21 420-439 98 98 98 180 180 180 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 100 100 100 183 183 183 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 100 100 100 183 183 183 F F F 29 580-599 100 100 100 183 183 183 T T T Max gen: 100, Max flush: 183 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 F F F 2 040-059 101 101 101 184 184 184 F F F 3 060-079 100 100 100 183 183 183 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 101 101 101 184 184 184 F F F 6 120-139 96 96 96 175 175 175 F F F 7 140-159 96 96 96 176 176 176 F F F 8 160-179 96 96 96 174 174 174 F F F 9 180-199 99 99 99 181 181 181 F F F 10 200-219 99 99 99 181 181 181 F F F 11 220-239 100 100 100 182 182 182 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 98 98 98 179 179 179 F F F 15 300-319 97 97 97 177 177 177 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 100 100 100 183 183 183 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 101 101 101 184 184 184 F F F 20 400-419 96 96 96 176 176 176 F F F 21 420-439 98 98 98 180 180 180 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 100 100 100 183 183 183 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 100 100 100 183 183 183 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 101, Max flush: 184 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.02199792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.066845801Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.067630473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.067683876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.067705213Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.067720981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} {"msg":"Crucible 1bc6f764-ca21-4a80-a232-6c1bb866e5d6 has session id: d2f5c7f4-7c64-492f-beb9-8042519a90bf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.068295492Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Upstairs opts: Upstairs UUID: 1bc6f764-ca21-4a80-a232-6c1bb866e5d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.068330127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Crucible stats registered with UUID: 1bc6f764-ca21-4a80-a232-6c1bb866e5d6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.068345865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:35.068358923Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.068869703Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.068902249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} {"msg":"1bc6f764-ca21-4a80-a232-6c1bb866e5d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.068949032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.06896596Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"0","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069003244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"0","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069030261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"1","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069057217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"1","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069085443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"2","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069128667Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"2","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069158103Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069188289Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069218025Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:35827)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069301623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:33900)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069464571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:55169)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.069496397Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.07005286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"0","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.070087895Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"2","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.070250133Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"1","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.070847641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"0","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.070878137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"2","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.070895094Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"1","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071448468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071558643Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[0]R flush_numbers[0..12]: [177, 176, 184, 183, 183, 184, 175, 176, 174, 181, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071602807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[0]R generation[0..12]: [97, 96, 101, 100, 100, 101, 96, 96, 96, 99, 99, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071622994Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071638192Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[1]R flush_numbers[0..12]: [177, 176, 184, 183, 183, 184, 175, 176, 174, 181, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.07165316Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[1]R generation[0..12]: [97, 96, 101, 100, 100, 101, 96, 96, 96, 99, 99, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071668188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071683206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[2]R flush_numbers[0..12]: [177, 176, 184, 183, 183, 184, 175, 176, 174, 181, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071697584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[2]R generation[0..12]: [97, 96, 101, 100, 100, 101, 96, 96, 96, 99, 99, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071711652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.07172611Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071740858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071764865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071789941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:07:35.071806039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"":"downstairs","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071840434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071862701Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"0","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071878209Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"1","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071892007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"client":"2","":"downstairs","session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071905585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071918154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"1bc6f764-ca21-4a80-a232-6c1bb866e5d6 is now active with session: d2f5c7f4-7c64-492f-beb9-8042519a90bf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071931442Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.07194382Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600,"session_id":"d2f5c7f4-7c64-492f-beb9-8042519a90bf"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071956518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} {"msg":"Activated sub_volume 1bc6f764-ca21-4a80-a232-6c1bb866e5d6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:35.071968187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 158 len 32768 data: 11 9 9 8 10 10 10 9 02/30 Write block 175 len 28672 data: 9 7 7 6 8 10 11 03/30 Read block 116 len 40960 04/30 Write block 356 len 4096 data: 11 05/30 Read block 476 len 40960 06/30 Read block 484 len 8192 07/30 Read block 33 len 20480 08/30 Write block 41 len 32768 data: 14 17 16 16 13 12 12 10 09/30 Write block 205 len 40960 data: 6 8 8 8 8 8 7 8 11 11 10/30 Write block 458 len 4096 data: 9 11/30 Read block 322 len 20480 12/30 Write block 394 len 24576 data: 5 6 8 8 7 6 13/30 Flush 14/30 Write block 446 len 36864 data: 12 15 17 17 15 15 12 11 9 15/30 Flush 16/30 Write block 563 len 28672 data: 9 10 10 9 8 9 8 17/30 Read block 128 len 40960 18/30 Write block 56 len 8192 data: 12 12 19/30 Read block 270 len 4096 20/30 Read block 592 len 16384 21/30 Read block 426 len 4096 22/30 Flush 23/30 Write block 122 len 28672 data: 13 10 12 13 13 9 9 24/30 Read block 572 len 40960 25/30 Write block 123 len 24576 data: 11 13 14 14 10 10 26/30 Read block 312 len 16384 27/30 Read block 501 len 4096 28/30 Read block 257 len 28672 29/30 Write block 425 len 28672 data: 11 11 10 9 10 9 7 30/30 Read block 310 len 28672 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:35.093402189Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 F F F 2 040-059 102 102 102 187 187 187 F F F 3 060-079 100 100 100 183 183 183 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 101 101 101 184 184 184 F F F 6 120-139 96 96 96 175 175 175 T T T 7 140-159 102 102 102 185 185 185 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 102 102 102 185 185 185 F F F 10 200-219 102 102 102 185 185 185 F F F 11 220-239 100 100 100 182 182 182 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 98 98 98 179 179 179 F F F 15 300-319 97 97 97 177 177 177 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 102 102 102 185 185 185 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 102 102 102 185 185 185 F F F 20 400-419 96 96 96 176 176 176 F F F 21 420-439 98 98 98 180 180 180 T T T 22 440-459 102 102 102 186 186 186 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 100 100 100 183 183 183 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 102 102 102 187 187 187 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 102, Max flush: 187 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 F F F 2 040-059 102 102 102 187 187 187 F F F 3 060-079 100 100 100 183 183 183 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 101 101 101 184 184 184 F F F 6 120-139 103 103 103 188 188 188 F F F 7 140-159 102 102 102 185 185 185 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 102 102 102 185 185 185 F F F 10 200-219 102 102 102 185 185 185 F F F 11 220-239 100 100 100 182 182 182 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 98 98 98 179 179 179 F F F 15 300-319 97 97 97 177 177 177 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 102 102 102 185 185 185 F F F 18 360-379 98 98 98 180 180 180 F F F 19 380-399 102 102 102 185 185 185 F F F 20 400-419 96 96 96 176 176 176 F F F 21 420-439 103 103 103 188 188 188 F F F 22 440-459 102 102 102 186 186 186 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 100 100 100 183 183 183 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 102 102 102 187 187 187 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 103, Max flush: 188 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.32957186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.375281244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.376149274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.376185089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.376199167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.376213955Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} {"msg":"Crucible ec14a035-545b-486b-940a-b1bf6ce3fe88 has session id: 3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.37683202Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Upstairs opts: Upstairs UUID: ec14a035-545b-486b-940a-b1bf6ce3fe88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.376868205Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Crucible stats registered with UUID: ec14a035-545b-486b-940a-b1bf6ce3fe88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.376883763Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:39.376896551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377359128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377388024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} {"msg":"ec14a035-545b-486b-940a-b1bf6ce3fe88 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377462513Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377480461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"0","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377509157Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"0","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377523725Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"1","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377538513Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"1","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377552461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"2","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377567419Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"2","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377581167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377595595Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377613353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"ds_connection connected from Ok(127.0.0.1:63136)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.37763696Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"ds_connection connected from Ok(127.0.0.1:58093)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.377736966Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"ds_connection connected from Ok(127.0.0.1:53884)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.37778004Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.378320855Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"1","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.378353581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"0","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.378993323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"1","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.37916014Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"0","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.379673399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.379796103Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.380076444Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"2","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.380596162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"2","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[0]R flush_numbers[0..12]: [177, 176, 187, 183, 183, 184, 188, 185, 185, 185, 185, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381094034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[0]R generation[0..12]: [97, 96, 102, 100, 100, 101, 103, 102, 102, 102, 102, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381115621Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381130999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[1]R flush_numbers[0..12]: [177, 176, 187, 183, 183, 184, 188, 185, 185, 185, 185, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381145617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[1]R generation[0..12]: [97, 96, 102, 100, 100, 101, 103, 102, 102, 102, 102, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381159885Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381174233Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[2]R flush_numbers[0..12]: [177, 176, 187, 183, 183, 184, 188, 185, 185, 185, 185, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381188511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[2]R generation[0..12]: [97, 96, 102, 100, 100, 101, 103, 102, 102, 102, 102, 100]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381207349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381222376Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381235925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381249633Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.38126978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381283428Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"":"downstairs","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381296916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381309215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"0","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381322853Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"1","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.38133704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"client":"2","":"downstairs","session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381350759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381363567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"ec14a035-545b-486b-940a-b1bf6ce3fe88 is now active with session: 3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381376785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381388964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608,"session_id":"3a6c252b-f79b-4aa9-b293-7bd7eb6f3aba"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.381401342Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} {"msg":"Activated sub_volume ec14a035-545b-486b-940a-b1bf6ce3fe88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:39.38141285Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 123 len 20480 data: 12 14 15 15 11 02/30 Write block 570 len 24576 data: 6 6 6 5 5 6 03/30 Read block 549 len 12288 04/30 Read block 264 len 8192 05/30 Flush 06/30 Write block 507 len 24576 data: 7 7 8 10 11 11 07/30 Write block 378 len 8192 data: 4 5 08/30 Read block 74 len 40960 09/30 Read block 533 len 32768 10/30 Write block 209 len 8192 data: 9 9 11/30 Read block 277 len 20480 12/30 Read block 127 len 36864 13/30 Read block 327 len 12288 14/30 Write block 439 len 28672 data: 2 4 6 6 9 10 11 15/30 Read block 485 len 12288 16/30 Write block 301 len 16384 data: 3 4 4 5 17/30 Read block 2 len 20480 18/30 Flush 19/30 Write block 64 len 28672 data: 12 12 9 9 7 7 7 20/30 Flush 21/30 Write block 227 len 40960 data: 12 14 12 13 11 13 11 9 10 12 22/30 Read block 305 len 8192 23/30 Write block 365 len 28672 data: 6 6 7 7 8 10 8 24/30 Read block 126 len 28672 25/30 Read block 235 len 40960 26/30 Write block 23 len 16384 data: 5 5 4 4 27/30 Write block 261 len 40960 data: 11 13 11 11 10 9 8 7 4 3 28/30 Write block 407 len 36864 data: 10 10 10 9 7 8 10 10 10 29/30 Read block 133 len 36864 30/30 Write block 260 len 32768 data: 14 12 14 12 12 11 10 9 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:39.402696093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 96 96 96 176 176 176 T T T 2 040-059 102 102 102 187 187 187 F F F 3 060-079 104 104 104 191 191 191 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 101 101 101 184 184 184 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 102 102 102 185 185 185 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 102 102 102 185 185 185 F F F 10 200-219 104 104 104 190 190 190 F F F 11 220-239 100 100 100 182 182 182 T T T 12 240-259 98 98 98 179 179 179 F F F 13 260-279 95 95 95 173 173 173 T T T 14 280-299 98 98 98 179 179 179 F F F 15 300-319 104 104 104 190 190 190 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 102 102 102 185 185 185 F F F 18 360-379 104 104 104 190 190 190 T T T 19 380-399 102 102 102 185 185 185 F F F 20 400-419 96 96 96 176 176 176 T T T 21 420-439 104 104 104 190 190 190 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 104, Max flush: 191 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 105 105 105 192 192 192 F F F 2 040-059 102 102 102 187 187 187 F F F 3 060-079 104 104 104 191 191 191 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 101 101 101 184 184 184 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 102 102 102 185 185 185 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 102 102 102 185 185 185 F F F 10 200-219 104 104 104 190 190 190 F F F 11 220-239 105 105 105 192 192 192 F F F 12 240-259 98 98 98 179 179 179 F F F 13 260-279 105 105 105 192 192 192 F F F 14 280-299 98 98 98 179 179 179 F F F 15 300-319 104 104 104 190 190 190 F F F 16 320-339 96 96 96 174 174 174 F F F 17 340-359 102 102 102 185 185 185 F F F 18 360-379 105 105 105 192 192 192 F F F 19 380-399 102 102 102 185 185 185 F F F 20 400-419 105 105 105 192 192 192 F F F 21 420-439 104 104 104 190 190 190 F F F 22 440-459 104 104 104 190 190 190 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 105, Max flush: 192 Begin loop 53 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.660476508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:53.705848767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.70641211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.738599833Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.739487891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.739531665Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.739550982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.73956613Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} {"msg":"Crucible d989fd09-3b4a-40ef-8b52-3c02f6f97b2e has session id: 16d97a63-95e2-41f7-a953-388bd1168b1d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740241047Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Upstairs opts: Upstairs UUID: d989fd09-3b4a-40ef-8b52-3c02f6f97b2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740282251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Crucible stats registered with UUID: d989fd09-3b4a-40ef-8b52-3c02f6f97b2e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740297659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:53.740310737Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740837045Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740863151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} {"msg":"d989fd09-3b4a-40ef-8b52-3c02f6f97b2e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740923493Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740951449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"0","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.740982155Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"0","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741023899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"1","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741045856Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"1","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741061624Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"2","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.74108694Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"2","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741116396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741147792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741176128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62170)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741385139Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36932)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.74160167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38868)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.741624726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.742297584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"1","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.742335149Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"2","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.742454052Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"0","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743020374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"1","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743048401Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"2","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743120061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"0","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743617232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743750694Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[0]R flush_numbers[0..12]: [177, 192, 187, 191, 183, 184, 189, 185, 185, 185, 190, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743818365Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[0]R generation[0..12]: [97, 105, 102, 104, 100, 101, 104, 102, 102, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743847211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743863619Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[1]R flush_numbers[0..12]: [177, 192, 187, 191, 183, 184, 189, 185, 185, 185, 190, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743878847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[1]R generation[0..12]: [97, 105, 102, 104, 100, 101, 104, 102, 102, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743893034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743907632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[2]R flush_numbers[0..12]: [177, 192, 187, 191, 183, 184, 189, 185, 185, 185, 190, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743921811Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[2]R generation[0..12]: [97, 105, 102, 104, 100, 101, 104, 102, 102, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743935819Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743950187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743963565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743977313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.743990651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744003989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"":"downstairs","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744028606Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744042364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"0","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744057052Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"1","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.74407091Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"client":"2","":"downstairs","session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744084728Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744097346Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"d989fd09-3b4a-40ef-8b52-3c02f6f97b2e is now active with session: 16d97a63-95e2-41f7-a953-388bd1168b1d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744110635Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744122803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616,"session_id":"16d97a63-95e2-41f7-a953-388bd1168b1d"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.744135371Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} {"msg":"Activated sub_volume d989fd09-3b4a-40ef-8b52-3c02f6f97b2e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:53.74414697Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 16384 data: 11 11 10 10 02/30 Write block 297 len 20480 data: 7 6 6 4 4 03/30 Flush 04/30 Write block 303 len 20480 data: 5 6 5 3 3 05/30 Write block 305 len 28672 data: 6 4 4 5 4 4 7 06/30 Flush 07/30 Write block 48 len 24576 data: 11 6 7 9 8 10 08/30 Write block 390 len 20480 data: 6 5 5 4 6 09/30 Read block 322 len 8192 10/30 Read block 59 len 12288 11/30 Read block 469 len 24576 12/30 Write block 441 len 12288 data: 7 7 10 13/30 Read block 247 len 8192 14/30 Read block 268 len 40960 15/30 Read block 38 len 36864 16/30 Read block 86 len 8192 17/30 Write block 298 len 8192 data: 7 7 18/30 Read block 39 len 4096 19/30 Write block 257 len 16384 data: 11 12 14 15 20/30 Write block 107 len 36864 data: 10 11 11 9 8 7 6 5 8 21/30 Write block 285 len 28672 data: 6 7 9 8 8 8 6 22/30 Read block 444 len 20480 23/30 Read block 294 len 4096 24/30 Read block 298 len 40960 25/30 Write block 421 len 32768 data: 7 10 9 10 12 12 11 10 26/30 Read block 487 len 28672 27/30 Read block 351 len 4096 28/30 Read block 337 len 12288 29/30 Read block 11 len 8192 30/30 Write block 318 len 36864 data: 6 6 5 6 6 7 7 8 6 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:53.764350001Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 105 105 105 192 192 192 F F F 2 040-059 102 102 102 187 187 187 T T T 3 060-079 104 104 104 191 191 191 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 101 101 101 184 184 184 T T T 6 120-139 104 104 104 189 189 189 F F F 7 140-159 106 106 106 193 193 193 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 102 102 102 185 185 185 F F F 10 200-219 104 104 104 190 190 190 F F F 11 220-239 105 105 105 192 192 192 F F F 12 240-259 98 98 98 179 179 179 T T T 13 260-279 105 105 105 192 192 192 T T T 14 280-299 106 106 106 193 193 193 T T T 15 300-319 106 106 106 194 194 194 T T T 16 320-339 96 96 96 174 174 174 T T T 17 340-359 102 102 102 185 185 185 F F F 18 360-379 105 105 105 192 192 192 F F F 19 380-399 102 102 102 185 185 185 T T T 20 400-419 105 105 105 192 192 192 F F F 21 420-439 104 104 104 190 190 190 T T T 22 440-459 104 104 104 190 190 190 T T T 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 106, Max flush: 194 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 105 105 105 192 192 192 F F F 2 040-059 107 107 107 195 195 195 F F F 3 060-079 104 104 104 191 191 191 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 107 107 107 195 195 195 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 106 106 106 193 193 193 F F F 8 160-179 102 102 102 185 185 185 F F F 9 180-199 102 102 102 185 185 185 F F F 10 200-219 104 104 104 190 190 190 F F F 11 220-239 105 105 105 192 192 192 F F F 12 240-259 107 107 107 195 195 195 F F F 13 260-279 107 107 107 195 195 195 F F F 14 280-299 107 107 107 195 195 195 F F F 15 300-319 107 107 107 195 195 195 F F F 16 320-339 107 107 107 195 195 195 F F F 17 340-359 102 102 102 185 185 185 F F F 18 360-379 105 105 105 192 192 192 F F F 19 380-399 107 107 107 195 195 195 F F F 20 400-419 105 105 105 192 192 192 F F F 21 420-439 107 107 107 195 195 195 F F F 22 440-459 107 107 107 195 195 195 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 98 98 98 179 179 179 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 107, Max flush: 195 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.069801981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.114058125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.114762568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.114806612Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.114824669Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.114843586Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} {"msg":"Crucible dc3d54d9-19a4-4344-bfaa-523d8e2dba8e has session id: 5cefd001-dde7-4b34-96f3-100644801c93","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.115446983Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Upstairs opts: Upstairs UUID: dc3d54d9-19a4-4344-bfaa-523d8e2dba8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.115493417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Crucible stats registered with UUID: dc3d54d9-19a4-4344-bfaa-523d8e2dba8e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.115518204Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:58.115532642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116045021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116070748Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} {"msg":"dc3d54d9-19a4-4344-bfaa-523d8e2dba8e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.11612707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116154746Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"0","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116184862Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"0","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.11620058Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116224197Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116246304Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"2","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116262871Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"2","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116276729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116291737Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116307595Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"ds_connection connected from Ok(127.0.0.1:48477)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116422589Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:58.116480072Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"ds_connection connected from Ok(127.0.0.1:63416)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116515856Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:58.116549962Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:58.116751404Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.116825634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.117241797Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"2","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.117269093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"0","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.117908765Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"2","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.118039777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"0","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.118476307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:58.118575703Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.117723766Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"ds_connection connected from Ok(127.0.0.1:62023)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.117791357Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.118981743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.119626685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[0]R flush_numbers[0..12]: [177, 192, 195, 191, 183, 195, 189, 193, 185, 185, 190, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.12024246Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[0]R generation[0..12]: [97, 105, 107, 104, 100, 107, 104, 106, 102, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120268836Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120285364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[1]R flush_numbers[0..12]: [177, 192, 195, 191, 183, 195, 189, 193, 185, 185, 190, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120300152Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[1]R generation[0..12]: [97, 105, 107, 104, 100, 107, 104, 106, 102, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.12031455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120329258Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[2]R flush_numbers[0..12]: [177, 192, 195, 191, 183, 195, 189, 193, 185, 185, 190, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120343756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[2]R generation[0..12]: [97, 105, 107, 104, 100, 107, 104, 106, 102, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120357934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120372312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.12038584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120399668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120413396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120427104Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"":"downstairs","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120444322Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.12045702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"0","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120470988Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"1","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120484876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"client":"2","":"downstairs","session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120498685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120511343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {Wait for a query_work_queue command to finish before sending IO "msg":"dc3d54d9-19a4-4344-bfaa-523d8e2dba8e is now active with session: 5cefd001-dde7-4b34-96f3-100644801c93","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.12053068Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120543768Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624,"session_id":"5cefd001-dde7-4b34-96f3-100644801c93"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120556386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} {"msg":"Activated sub_volume dc3d54d9-19a4-4344-bfaa-523d8e2dba8e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:08.120568065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 28672 data: 6 5 5 8 8 8 9 02/30 Read block 528 len 28672 03/30 Write block 111 len 20480 data: 9 8 7 6 9 04/30 Flush 05/30 Write block 222 len 4096 data: 10 06/30 Read block 205 len 8192 07/30 Write block 258 len 20480 data: 13 15 16 13 15 08/30 Write block 274 len 32768 data: 4 6 5 5 6 6 9 8 09/30 Read block 115 len 16384 10/30 Write block 256 len 16384 data: 11 12 14 16 11/30 Read block 168 len 8192 12/30 Write block 178 len 36864 data: 7 9 11 12 10 9 7 8 8 13/30 Read block 19 len 20480 14/30 Read block 134 len 28672 15/30 Read block 201 len 32768 16/30 Read block 100 len 36864 17/30 Read block 252 len 4096 18/30 Write block 312 len 40960 data: 9 9 10 8 8 6 7 7 6 7 19/30 Read block 224 len 24576 20/30 Write block 440 len 36864 data: 5 8 8 11 11 12 13 16 18 21/30 Write block 116 len 28672 data: 10 11 11 15 13 15 14 22/30 Read block 333 len 12288 23/30 Read block 67 len 4096 24/30 Write block 431 len 8192 data: 8 5 25/30 Write block 74 len 4096 data: 4 26/30 Write block 330 len 40960 data: 8 9 7 5 6 7 10 9 11 10 27/30 Read block 269 len 32768 28/30 Write block 529 len 8192 data: 12 12 29/30 Read block 3 len 16384 30/30 Write block 388 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 1 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 25 24 25 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:08.14146931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 105 105 105 192 192 192 F F F 2 040-059 107 107 107 195 195 195 F F F 3 060-079 104 104 104 191 191 191 T T T 4 080-099 100 100 100 183 183 183 F F F 5 100-119 108 108 108 196 196 196 T T T 6 120-139 104 104 104 189 189 189 T T T 7 140-159 106 106 106 193 193 193 F F F 8 160-179 102 102 102 185 185 185 T T T 9 180-199 102 102 102 185 185 185 T T T 10 200-219 104 104 104 190 190 190 F F F 11 220-239 105 105 105 192 192 192 T T T 12 240-259 107 107 107 195 195 195 T T T 13 260-279 107 107 107 195 195 195 T T T 14 280-299 107 107 107 195 195 195 T T T 15 300-319 108 108 108 196 196 196 T T T 16 320-339 107 107 107 195 195 195 T T T 17 340-359 102 102 102 185 185 185 F F F 18 360-379 105 105 105 192 192 192 F F F 19 380-399 107 107 107 195 195 195 T T T 20 400-419 105 105 105 192 192 192 F F F 21 420-439 107 107 107 195 195 195 T T T 22 440-459 107 107 107 195 195 195 T T T 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 98 98 98 179 179 179 T T T 27 540-559 93 93 93 170 170 170 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 108, Max flush: 196 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 177 177 177 F F F 1 020-039 105 105 105 192 192 192 F F F 2 040-059 107 107 107 195 195 195 F F F 3 060-079 109 109 109 197 197 197 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 109 109 109 197 197 197 F F F 7 140-159 106 106 106 193 193 193 F F F 8 160-179 109 109 109 197 197 197 F F F 9 180-199 109 109 109 197 197 197 F F F 10 200-219 104 104 104 190 190 190 F F F 11 220-239 109 109 109 197 197 197 F F F 12 240-259 109 109 109 197 197 197 F F F 13 260-279 109 109 109 197 197 197 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 109 109 109 197 197 197 F F F 16 320-339 109 109 109 197 197 197 F F F 17 340-359 102 102 102 185 185 185 F F F 18 360-379 105 105 105 192 192 192 F F F 19 380-399 109 109 109 197 197 197 F F F 20 400-419 105 105 105 192 192 192 F F F 21 420-439 109 109 109 197 197 197 F F F 22 440-459 109 109 109 197 197 197 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 109 109 109 197 197 197 F F F 27 540-559 93 93 93 170 170 170 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 109, Max flush: 197 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.446699007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:22.48949994Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.489957557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.521632141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.522333164Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.522370489Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.522386097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.522400985Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} {"msg":"Crucible b66e8233-a58b-4620-bfb5-7692a157be6e has session id: c3c3d256-ae44-4aab-b671-d71e47fdf904","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523081441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Upstairs opts: Upstairs UUID: b66e8233-a58b-4620-bfb5-7692a157be6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523127355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Crucible stats registered with UUID: b66e8233-a58b-4620-bfb5-7692a157be6e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523151492Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:22.523169309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523558816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523582763Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} {"msg":"b66e8233-a58b-4620-bfb5-7692a157be6e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523632605Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523659622Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"0","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523689868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"0","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523705446Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"1","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523720304Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"1","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523734022Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"2","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.52374862Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"2","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523768007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523797553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523823109Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"ds_connection connected from Ok(127.0.0.1:38542)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523850386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"ds_connection connected from Ok(127.0.0.1:53868)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523877122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"ds_connection connected from Ok(127.0.0.1:38318)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.523902779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.524580185Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"1","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.52462024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"2","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.52491127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"0","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.525431958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"1","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.525469123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"2","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.5254949Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"0","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.525980543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526104685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[0]R flush_numbers[0..12]: [177, 192, 195, 197, 183, 197, 197, 193, 197, 197, 190, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526231078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[0]R generation[0..12]: [97, 105, 107, 109, 100, 109, 109, 106, 109, 109, 104, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526251076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526269323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[1]R flush_numbers[0..12]: [177, 192, 195, 197, 183, 197, 197, 193, 197, 197, 190, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526283971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[1]R generation[0..12]: [97, 105, 107, 109, 100, 109, 109, 106, 109, 109, 104, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526298149Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526312457Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[2]R flush_numbers[0..12]: [177, 192, 195, 197, 183, 197, 197, 193, 197, 197, 190, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526326395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[2]R generation[0..12]: [97, 105, 107, 109, 100, 109, 109, 106, 109, 109, 104, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526340383Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526355121Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526368859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526382497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526401695Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"All extents match","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526423562Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"":"downstairs","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.52643751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526449788Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"0","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526463396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"1","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526479614Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"client":"2","":"downstairs","session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526493502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.52650621Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"b66e8233-a58b-4620-bfb5-7692a157be6e is now active with session: c3c3d256-ae44-4aab-b671-d71e47fdf904","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526519288Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526531477Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632,"session_id":"c3c3d256-ae44-4aab-b671-d71e47fdf904"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526543925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} {"msg":"Activated sub_volume b66e8233-a58b-4620-bfb5-7692a157be6e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:22.526555383Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 69 len 20480 data: 8 8 7 5 6 02/30 Read block 326 len 16384 03/30 Read block 98 len 40960 04/30 Write block 535 len 36864 data: 9 9 9 8 10 8 6 6 6 05/30 Write block 502 len 28672 data: 8 11 11 9 10 8 8 06/30 Flush 07/30 Write block 263 len 8192 data: 13 13 08/30 Write block 18 len 36864 data: 10 12 8 6 5 6 6 5 5 09/30 Write block 524 len 40960 data: 8 9 12 9 11 13 13 12 12 12 10/30 Write block 213 len 8192 data: 12 12 11/30 Read block 55 len 12288 12/30 Read block 565 len 8192 13/30 Flush 14/30 Write block 17 len 20480 data: 9 11 13 9 7 15/30 Write block 341 len 28672 data: 8 11 10 13 12 10 11 16/30 Read block 146 len 28672 17/30 Read block 433 len 32768 18/30 Read block 157 len 8192 19/30 Write block 434 len 24576 data: 6 5 4 3 2 3 20/30 Write block 404 len 16384 data: 11 9 8 11 21/30 Write block 410 len 12288 data: 10 8 9 22/30 Read block 481 len 32768 23/30 Read block 256 len 12288 24/30 Read block 203 len 24576 25/30 Write block 457 len 8192 data: 9 10 26/30 Flush 27/30 Write block 46 len 40960 data: 13 13 12 7 8 10 9 11 10 13 28/30 Write block 451 len 8192 data: 16 13 29/30 Write block 207 len 20480 data: 9 9 10 10 8 30/30 Read block 235 len 24576 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:22.54642722Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 200 200 200 F F F 1 020-039 110 110 110 200 200 200 F F F 2 040-059 107 107 107 195 195 195 T T T 3 060-079 110 110 110 198 198 198 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 109 109 109 197 197 197 F F F 7 140-159 106 106 106 193 193 193 F F F 8 160-179 109 109 109 197 197 197 F F F 9 180-199 109 109 109 197 197 197 F F F 10 200-219 110 110 110 199 199 199 T T T 11 220-239 109 109 109 197 197 197 F F F 12 240-259 109 109 109 197 197 197 F F F 13 260-279 110 110 110 199 199 199 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 109 109 109 197 197 197 F F F 16 320-339 109 109 109 197 197 197 F F F 17 340-359 110 110 110 200 200 200 F F F 18 360-379 105 105 105 192 192 192 F F F 19 380-399 109 109 109 197 197 197 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 110 110 110 200 200 200 F F F 22 440-459 110 110 110 200 200 200 T T T 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 110 110 110 198 198 198 F F F 26 520-539 110 110 110 199 199 199 F F F 27 540-559 110 110 110 198 198 198 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 110, Max flush: 200 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 200 200 200 F F F 1 020-039 110 110 110 200 200 200 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 110 110 110 198 198 198 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 109 109 109 197 197 197 F F F 7 140-159 106 106 106 193 193 193 F F F 8 160-179 109 109 109 197 197 197 F F F 9 180-199 109 109 109 197 197 197 F F F 10 200-219 111 111 111 201 201 201 F F F 11 220-239 109 109 109 197 197 197 F F F 12 240-259 109 109 109 197 197 197 F F F 13 260-279 110 110 110 199 199 199 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 109 109 109 197 197 197 F F F 16 320-339 109 109 109 197 197 197 F F F 17 340-359 110 110 110 200 200 200 F F F 18 360-379 105 105 105 192 192 192 F F F 19 380-399 109 109 109 197 197 197 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 110 110 110 200 200 200 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 90 90 90 165 165 165 F F F 24 480-499 100 100 100 183 183 183 F F F 25 500-519 110 110 110 198 198 198 F F F 26 520-539 110 110 110 199 199 199 F F F 27 540-559 110 110 110 198 198 198 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 111, Max flush: 201 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.813926163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.861264642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.86214685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.862182525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.862197023Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.862211671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} {"msg":"Crucible c9a0a9d0-9816-4880-b41b-1f9a50d60a28 has session id: d08ed896-885e-4be0-bae0-4ea3cd1b119b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.862833056Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Upstairs opts: Upstairs UUID: c9a0a9d0-9816-4880-b41b-1f9a50d60a28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.862866441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Crucible stats registered with UUID: c9a0a9d0-9816-4880-b41b-1f9a50d60a28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.862881499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:26.862894437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863371782Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863405537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} {"msg":"c9a0a9d0-9816-4880-b41b-1f9a50d60a28 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863463649Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863490985Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"0","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863521361Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"0","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863540299Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"1","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863555466Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"1","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863569185Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"2","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863583843Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"2","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863607369Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863635055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863664091Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42259)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863701437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60430)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863728243Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45790)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.863754319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.864579846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"0","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.864609552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"1","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.865079707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"2","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.865209299Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"0","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.865242194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"1","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.865688603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"2","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.865828504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.865846442Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[0]R flush_numbers[0..12]: [200, 200, 201, 198, 183, 197, 197, 193, 197, 197, 201, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866180216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[0]R generation[0..12]: [110, 110, 111, 110, 100, 109, 109, 106, 109, 109, 111, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866204812Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.86622114Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[1]R flush_numbers[0..12]: [200, 200, 201, 198, 183, 197, 197, 193, 197, 197, 201, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866235948Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[1]R generation[0..12]: [110, 110, 111, 110, 100, 109, 109, 106, 109, 109, 111, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866250236Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866264744Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[2]R flush_numbers[0..12]: [200, 200, 201, 198, 183, 197, 197, 193, 197, 197, 201, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866278932Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[2]R generation[0..12]: [110, 110, 111, 110, 100, 109, 109, 106, 109, 109, 111, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.86629293Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866309808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866326685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:26.866348382Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866372049Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866389097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"":"downstairs","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866406334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866421982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"0","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.86643937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"1","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866456307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"client":"2","":"downstairs","session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866473305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866488863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"c9a0a9d0-9816-4880-b41b-1f9a50d60a28 is now active with session: d08ed896-885e-4be0-bae0-4ea3cd1b119b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866505091Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866520349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640,"session_id":"d08ed896-885e-4be0-bae0-4ea3cd1b119b"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866535847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} {"msg":"Activated sub_volume c9a0a9d0-9816-4880-b41b-1f9a50d60a28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:26.866550115Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 360 len 12288 data: 6 6 3 02/30 Write block 465 len 36864 data: 6 7 6 8 8 9 11 11 11 03/30 Read block 432 len 24576 04/30 Write block 477 len 36864 data: 8 6 7 7 7 7 10 13 12 05/30 Read block 187 len 8192 06/30 Write block 270 len 40960 data: 4 4 3 6 5 7 6 6 7 7 07/30 Read block 0 len 24576 08/30 Read block 122 len 8192 09/30 Flush 10/30 Write block 193 len 36864 data: 9 8 7 8 6 8 8 8 7 11/30 Write block 134 len 20480 data: 9 12 9 9 8 12/30 Read block 504 len 24576 13/30 Read block 255 len 24576 14/30 Write block 330 len 28672 data: 9 10 8 6 7 8 11 15/30 Read block 277 len 4096 16/30 Read block 232 len 12288 17/30 Flush 18/30 Write block 362 len 40960 data: 4 4 5 7 7 8 8 9 11 9 19/30 Read block 313 len 32768 20/30 Flush 21/30 Flush 22/30 Write block 461 len 24576 data: 5 4 4 5 7 8 23/30 Read block 360 len 8192 24/30 Write block 6 len 28672 data: 5 5 4 5 8 8 9 25/30 Write block 333 len 4096 data: 7 26/30 Read block 272 len 32768 27/30 Write block 508 len 16384 data: 9 9 11 12 28/30 Write block 158 len 32768 data: 12 10 10 9 11 11 11 10 29/30 Write block 165 len 16384 data: 11 6 8 11 30/30 Read block 301 len 12288 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 3 4 Done 9 8 6 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:26.888342947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 200 200 200 T T T 1 020-039 110 110 110 200 200 200 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 110 110 110 198 198 198 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 106 106 106 193 193 193 T T T 8 160-179 109 109 109 197 197 197 T T T 9 180-199 112 112 112 203 203 203 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 109 109 109 197 197 197 F F F 12 240-259 109 109 109 197 197 197 F F F 13 260-279 112 112 112 202 202 202 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 109 109 109 197 197 197 F F F 16 320-339 112 112 112 203 203 203 T T T 17 340-359 110 110 110 200 200 200 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 109 109 109 197 197 197 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 110 110 110 200 200 200 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 112 112 112 202 202 202 T T T 24 480-499 112 112 112 202 202 202 F F F 25 500-519 110 110 110 198 198 198 T T T 26 520-539 110 110 110 199 199 199 F F F 27 540-559 110 110 110 198 198 198 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 112, Max flush: 204 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 205 205 205 F F F 1 020-039 110 110 110 200 200 200 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 110 110 110 198 198 198 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 113 113 113 205 205 205 F F F 8 160-179 113 113 113 205 205 205 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 109 109 109 197 197 197 F F F 12 240-259 109 109 109 197 197 197 F F F 13 260-279 112 112 112 202 202 202 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 109 109 109 197 197 197 F F F 16 320-339 113 113 113 205 205 205 F F F 17 340-359 110 110 110 200 200 200 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 109 109 109 197 197 197 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 110 110 110 200 200 200 F F F 22 440-459 111 111 111 201 201 201 F F F 23 460-479 113 113 113 205 205 205 F F F 24 480-499 112 112 112 202 202 202 F F F 25 500-519 113 113 113 205 205 205 F F F 26 520-539 110 110 110 199 199 199 F F F 27 540-559 110 110 110 198 198 198 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 113, Max flush: 205 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.14057301Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.185633192Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.18630417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.186346594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.186364412Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.186382769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} {"msg":"Crucible d5d11f91-14db-452c-b058-28c38778268a has session id: 46dcbe8a-717a-4a5d-a7d3-b62e350873ba","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.186944512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Upstairs opts: Upstairs UUID: d5d11f91-14db-452c-b058-28c38778268a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.186987946Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Crucible stats registered with UUID: d5d11f91-14db-452c-b058-28c38778268a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187014912Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:31.18702924Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187432685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187454871Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} {"msg":"d5d11f91-14db-452c-b058-28c38778268a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187524792Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.18754207Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"0","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187577814Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"0","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187603411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"1","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187694289Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"1","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187726074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"2","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.18775652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"2","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187784286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187814242Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187839019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:54297)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187863125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:57890)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187895421Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:41222)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.187926846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.188490089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"0","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.188521635Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"1","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.189172415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"1","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.189325954Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"0","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.18972129Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.189787371Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"2","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.189904225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190292251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"2","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[0]R flush_numbers[0..12]: [205, 200, 201, 198, 183, 197, 203, 205, 205, 203, 203, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190813229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[0]R generation[0..12]: [113, 110, 111, 110, 100, 109, 112, 113, 113, 112, 112, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190838566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190857234Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[1]R flush_numbers[0..12]: [205, 200, 201, 198, 183, 197, 203, 205, 205, 203, 203, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190875431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[1]R generation[0..12]: [113, 110, 111, 110, 100, 109, 112, 113, 113, 112, 112, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190898478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190917455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[2]R flush_numbers[0..12]: [205, 200, 201, 198, 183, 197, 203, 205, 205, 203, 203, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190935363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[2]R generation[0..12]: [113, 110, 111, 110, 100, 109, 112, 113, 113, 112, 112, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.19095272Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190970948Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Max found gen is 114","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.190994015Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191011102Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.19102774Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191044568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"":"downstairs","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191061415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191076683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"0","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191093681Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"1","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191110869Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"client":"2","":"downstairs","session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191127716Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191143434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"d5d11f91-14db-452c-b058-28c38778268a is now active with session: 46dcbe8a-717a-4a5d-a7d3-b62e350873ba","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191162101Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191177899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648,"session_id":"46dcbe8a-717a-4a5d-a7d3-b62e350873ba"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191193597Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} {"msg":"Activated sub_volume d5d11f91-14db-452c-b058-28c38778268a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:31.191207975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 500 len 16384 data: 10 8 9 12 02/30 Write block 527 len 40960 data: 10 12 14 14 13 13 13 12 10 10 03/30 Write block 428 len 8192 data: 11 11 04/30 Read block 322 len 36864 05/30 Write block 318 len 4096 data: 8 06/30 Write block 8 len 24576 data: 5 6 9 9 10 10 07/30 Write block 228 len 8192 data: 15 13 08/30 Flush 09/30 Write block 179 len 16384 data: 10 12 13 11 10/30 Read block 149 len 12288 11/30 Write block 131 len 28672 data: 8 8 10 10 13 10 10 12/30 Read block 322 len 8192 13/30 Read block 299 len 32768 14/30 Read block 528 len 4096 15/30 Flush 16/30 Flush 17/30 Write block 69 len 4096 data: 9 18/30 Read block 429 len 8192 19/30 Read block 463 len 24576 20/30 Read block 158 len 20480 21/30 Write block 386 len 20480 data: 6 6 8 5 7 22/30 Write block 236 len 4096 data: 13 23/30 Read block 396 len 16384 24/30 Write block 455 len 28672 data: 9 8 10 11 8 5 6 25/30 Read block 569 len 16384 26/30 Read block 80 len 40960 27/30 Read block 381 len 16384 28/30 Write block 531 len 36864 data: 14 14 14 13 11 11 10 9 11 29/30 Read block 510 len 32768 30/30 Read block 388 len 12288 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 1 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:31.211538039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 206 206 206 F F F 1 020-039 110 110 110 200 200 200 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 110 110 110 198 198 198 T T T 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 114 114 114 207 207 207 F F F 7 140-159 113 113 113 205 205 205 F F F 8 160-179 114 114 114 207 207 207 F F F 9 180-199 114 114 114 207 207 207 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 114 114 114 206 206 206 T T T 12 240-259 109 109 109 197 197 197 F F F 13 260-279 112 112 112 202 202 202 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 113 113 113 205 205 205 F F F 17 340-359 110 110 110 200 200 200 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 109 109 109 197 197 197 T T T 20 400-419 110 110 110 200 200 200 F F F 21 420-439 114 114 114 206 206 206 F F F 22 440-459 111 111 111 201 201 201 T T T 23 460-479 113 113 113 205 205 205 T T T 24 480-499 112 112 112 202 202 202 F F F 25 500-519 114 114 114 206 206 206 F F F 26 520-539 114 114 114 206 206 206 T T T 27 540-559 110 110 110 198 198 198 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 114, Max flush: 207 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 206 206 206 F F F 1 020-039 110 110 110 200 200 200 F F F 2 040-059 111 111 111 201 201 201 F F F 3 060-079 115 115 115 208 208 208 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 114 114 114 207 207 207 F F F 7 140-159 113 113 113 205 205 205 F F F 8 160-179 114 114 114 207 207 207 F F F 9 180-199 114 114 114 207 207 207 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 115 115 115 208 208 208 F F F 12 240-259 109 109 109 197 197 197 F F F 13 260-279 112 112 112 202 202 202 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 113 113 113 205 205 205 F F F 17 340-359 110 110 110 200 200 200 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 115 115 115 208 208 208 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 114 114 114 206 206 206 F F F 22 440-459 115 115 115 208 208 208 F F F 23 460-479 115 115 115 208 208 208 F F F 24 480-499 112 112 112 202 202 202 F F F 25 500-519 114 114 114 206 206 206 F F F 26 520-539 115 115 115 208 208 208 F F F 27 540-559 110 110 110 198 198 198 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 115, Max flush: 208 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.462384732Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.506640505Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.507340839Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.507385683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.5074031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.507421578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} {"msg":"Crucible 2ccbc85e-d9b8-48a6-a2fa-4a80cf93cecb has session id: 6360f74f-ac6d-469e-bca6-86c98c5936ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508000688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Upstairs opts: Upstairs UUID: 2ccbc85e-d9b8-48a6-a2fa-4a80cf93cecb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508054481Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Crucible stats registered with UUID: 2ccbc85e-d9b8-48a6-a2fa-4a80cf93cecb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508082847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:35.508100934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508526176Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508551082Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} {"msg":"2ccbc85e-d9b8-48a6-a2fa-4a80cf93cecb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508623343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.50864106Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"0","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508669796Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"0","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508684964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508704981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508722589Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"2","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508748225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"2","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508774272Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508802218Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508830204Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:33366)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.508864849Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:35.508910273Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:35.508941179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:56105)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.509044234Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:35.509085019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.50922275Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.509722701Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"2","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.509751637Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"0","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.510404697Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"2","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.510438853Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"0","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.51104327Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:35.511068536Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.511114135Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:39805)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.511286141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.512335457Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.512877512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[0]R flush_numbers[0..12]: [206, 200, 201, 208, 183, 197, 207, 205, 207, 207, 203, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513489868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[0]R generation[0..12]: [114, 110, 111, 115, 100, 109, 114, 113, 114, 114, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513514815Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513534082Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[1]R flush_numbers[0..12]: [206, 200, 201, 208, 183, 197, 207, 205, 207, 207, 203, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.51355201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[1]R generation[0..12]: [114, 110, 111, 115, 100, 109, 114, 113, 114, 114, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513571647Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513590824Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[2]R flush_numbers[0..12]: [206, 200, 201, 208, 183, 197, 207, 205, 207, 207, 203, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513614231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[2]R generation[0..12]: [114, 110, 111, 115, 100, 109, 114, 113, 114, 114, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513633008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513651016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513667754Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513684721Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513701329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513718157Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"":"downstairs","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513735214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513750412Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"0","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.51376751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"1","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513784527Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"client":"2","":"downstairs","session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513807974Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513823872Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"2ccbc85e-d9b8-48a6-a2fa-4a80cf93cecb is now active with session: 6360f74f-ac6d-469e-bca6-86c98c5936ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.51384049Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513855878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656,"session_id":"6360f74f-ac6d-469e-bca6-86c98c5936ad"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513873935Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} {"msg":"Activated sub_volume 2ccbc85e-d9b8-48a6-a2fa-4a80cf93cecb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:45.513888473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 238 len 40960 data: 9 11 12 12 10 11 11 12 12 12 02/30 Write block 236 len 24576 data: 14 12 10 12 13 13 03/30 Read block 209 len 4096 04/30 Read block 497 len 24576 05/30 Read block 434 len 4096 06/30 Write block 534 len 20480 data: 14 12 12 11 10 07/30 Read block 304 len 12288 08/30 Read block 212 len 12288 09/30 Write block 41 len 24576 data: 15 18 17 17 14 14 10/30 Flush 11/30 Write block 266 len 4096 data: 11 12/30 Read block 43 len 12288 13/30 Write block 189 len 36864 data: 10 10 12 10 10 9 8 9 7 14/30 Read block 433 len 12288 15/30 Read block 213 len 24576 16/30 Flush 17/30 Write block 539 len 12288 data: 12 9 7 18/30 Read block 432 len 28672 19/30 Read block 566 len 20480 20/30 Read block 531 len 12288 21/30 Write block 249 len 20480 data: 11 10 6 5 5 22/30 Read block 76 len 32768 23/30 Read block 421 len 4096 24/30 Write block 32 len 12288 data: 5 6 4 25/30 Read block 162 len 8192 26/30 Write block 143 len 4096 data: 7 27/30 Read block 361 len 28672 28/30 Read block 231 len 24576 29/30 Read block 583 len 24576 30/30 Read block 354 len 16384 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 13 14 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:45.534786759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 206 206 206 F F F 1 020-039 110 110 110 200 200 200 T T T 2 040-059 116 116 116 209 209 209 F F F 3 060-079 115 115 115 208 208 208 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 114 114 114 207 207 207 F F F 7 140-159 113 113 113 205 205 205 T T T 8 160-179 114 114 114 207 207 207 F F F 9 180-199 116 116 116 210 210 210 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 116 116 116 209 209 209 F F F 12 240-259 116 116 116 209 209 209 T T T 13 260-279 116 116 116 210 210 210 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 113 113 113 205 205 205 F F F 17 340-359 110 110 110 200 200 200 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 115 115 115 208 208 208 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 114 114 114 206 206 206 F F F 22 440-459 115 115 115 208 208 208 F F F 23 460-479 115 115 115 208 208 208 F F F 24 480-499 112 112 112 202 202 202 F F F 25 500-519 114 114 114 206 206 206 F F F 26 520-539 116 116 116 209 209 209 T T T 27 540-559 110 110 110 198 198 198 T T T 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 116, Max flush: 210 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 206 206 206 F F F 1 020-039 117 117 117 211 211 211 F F F 2 040-059 116 116 116 209 209 209 F F F 3 060-079 115 115 115 208 208 208 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 114 114 114 207 207 207 F F F 7 140-159 117 117 117 211 211 211 F F F 8 160-179 114 114 114 207 207 207 F F F 9 180-199 116 116 116 210 210 210 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 116 116 116 209 209 209 F F F 12 240-259 117 117 117 211 211 211 F F F 13 260-279 116 116 116 210 210 210 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 113 113 113 205 205 205 F F F 17 340-359 110 110 110 200 200 200 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 115 115 115 208 208 208 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 114 114 114 206 206 206 F F F 22 440-459 115 115 115 208 208 208 F F F 23 460-479 115 115 115 208 208 208 F F F 24 480-499 112 112 112 202 202 202 F F F 25 500-519 114 114 114 206 206 206 F F F 26 520-539 117 117 117 211 211 211 F F F 27 540-559 117 117 117 211 211 211 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 117, Max flush: 211 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.784448504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.828430315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.829214507Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.829249472Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.829269529Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.829284507Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} {"msg":"Crucible c16c96a1-1558-4fee-9459-182bbe208504 has session id: b4e348ea-5818-4fd3-90a6-a8c662cf12c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.829891714Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Upstairs opts: Upstairs UUID: c16c96a1-1558-4fee-9459-182bbe208504, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.829935977Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Crucible stats registered with UUID: c16c96a1-1558-4fee-9459-182bbe208504","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.829951245Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:49.829964424Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830475213Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.83050219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} {"msg":"c16c96a1-1558-4fee-9459-182bbe208504 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830563761Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830588627Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"0","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830617144Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"0","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830632122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"1","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.83064691Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"1","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830668947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830706431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830731268Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830783341Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830813927Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:49.830852851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:49.830907874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40764)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830938799Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58982)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.830968575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:49.830998891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.83108234Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.831643702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"0","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.831671749Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"1","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.832357054Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"1","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.83238672Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"0","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.832983888Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:49.833009425Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.832319086Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62095)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.832407794Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.833489945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.833987137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[0]R flush_numbers[0..12]: [206, 211, 209, 208, 183, 197, 207, 211, 207, 210, 203, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834554708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[0]R generation[0..12]: [114, 117, 116, 115, 100, 109, 114, 117, 114, 116, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834585314Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834605321Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[1]R flush_numbers[0..12]: [206, 211, 209, 208, 183, 197, 207, 211, 207, 210, 203, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834623709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[1]R generation[0..12]: [114, 117, 116, 115, 100, 109, 114, 117, 114, 116, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834641767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834659934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[2]R flush_numbers[0..12]: [206, 211, 209, 208, 183, 197, 207, 211, 207, 210, 203, 209]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834677772Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[2]R generation[0..12]: [114, 117, 116, 115, 100, 109, 114, 117, 114, 116, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834695619Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834720026Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834737723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834755421Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834772629Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834789576Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"":"downstairs","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:08:59.834806864Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834829091Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"0","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834846588Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"1","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834863996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"client":"2","":"downstairs","session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834881344Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834897012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"c16c96a1-1558-4fee-9459-182bbe208504 is now active with session: b4e348ea-5818-4fd3-90a6-a8c662cf12c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834913539Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834928977Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664,"session_id":"b4e348ea-5818-4fd3-90a6-a8c662cf12c5"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834944455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} {"msg":"Activated sub_volume c16c96a1-1558-4fee-9459-182bbe208504","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.834958623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 237 len 8192 data: 13 11 02/30 Read block 546 len 8192 03/30 Flush 04/30 Write block 2 len 36864 data: 2 3 3 4 6 6 6 7 10 05/30 Read block 473 len 4096 06/30 Read block 177 len 16384 07/30 Write block 337 len 28672 data: 10 12 11 11 9 12 11 08/30 Write block 482 len 4096 data: 8 09/30 Write block 421 len 28672 data: 8 11 10 11 13 13 12 10/30 Write block 265 len 32768 data: 12 12 10 8 5 5 5 4 11/30 Write block 253 len 20480 data: 6 7 11 12 13 12/30 Read block 193 len 20480 13/30 Write block 512 len 28672 data: 12 13 15 16 13 10 9 14/30 Write block 456 len 12288 data: 9 11 12 15/30 Read block 109 len 8192 16/30 Read block 60 len 4096 17/30 Read block 506 len 8192 18/30 Read block 12 len 28672 19/30 Write block 490 len 36864 data: 9 8 8 8 9 11 10 10 10 20/30 Read block 406 len 16384 21/30 Flush 22/30 Write block 547 len 20480 data: 7 8 8 9 10 23/30 Read block 30 len 8192 24/30 Write block 537 len 36864 data: 12 11 13 10 8 7 7 5 7 25/30 Read block 243 len 8192 26/30 Write block 54 len 8192 data: 11 14 27/30 Write block 344 len 12288 data: 14 13 11 28/30 Write block 125 len 36864 data: 16 16 12 11 9 8 9 9 11 29/30 Read block 155 len 8192 30/30 Read block 40 len 24576 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:59.855734495Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 117 117 117 211 211 211 F F F 2 040-059 116 116 116 209 209 209 T T T 3 060-079 115 115 115 208 208 208 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 114 114 114 207 207 207 T T T 7 140-159 117 117 117 211 211 211 F F F 8 160-179 114 114 114 207 207 207 F F F 9 180-199 116 116 116 210 210 210 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 118 118 118 213 213 213 F F F 13 260-279 118 118 118 213 213 213 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 118 118 118 213 213 213 F F F 17 340-359 118 118 118 213 213 213 T T T 18 360-379 112 112 112 204 204 204 F F F 19 380-399 115 115 115 208 208 208 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 118 118 118 213 213 213 F F F 22 440-459 118 118 118 213 213 213 F F F 23 460-479 115 115 115 208 208 208 F F F 24 480-499 118 118 118 213 213 213 F F F 25 500-519 118 118 118 213 213 213 F F F 26 520-539 117 117 117 211 211 211 T T T 27 540-559 117 117 117 211 211 211 T T T 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 118, Max flush: 213 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 117 117 117 211 211 211 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 115 115 115 208 208 208 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 119 119 119 214 214 214 F F F 7 140-159 117 117 117 211 211 211 F F F 8 160-179 114 114 114 207 207 207 F F F 9 180-199 116 116 116 210 210 210 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 118 118 118 213 213 213 F F F 13 260-279 118 118 118 213 213 213 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 118 118 118 213 213 213 F F F 17 340-359 119 119 119 214 214 214 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 115 115 115 208 208 208 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 118 118 118 213 213 213 F F F 22 440-459 118 118 118 213 213 213 F F F 23 460-479 115 115 115 208 208 208 F F F 24 480-499 118 118 118 213 213 213 F F F 25 500-519 118 118 118 213 213 213 F F F 26 520-539 119 119 119 214 214 214 F F F 27 540-559 119 119 119 214 214 214 F F F 28 560-579 104 104 104 189 189 189 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 119, Max flush: 214 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.110083516Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:04.154818753Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.155388515Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.187595465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.188323275Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.188367429Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.188390626Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.188409443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} {"msg":"Crucible c69345e5-f150-4f5d-a8b6-98e5f7b81df5 has session id: 4220b028-8416-4033-8b31-6b2fa43fa956","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.188960477Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Upstairs opts: Upstairs UUID: c69345e5-f150-4f5d-a8b6-98e5f7b81df5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189030658Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Crucible stats registered with UUID: c69345e5-f150-4f5d-a8b6-98e5f7b81df5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189052865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:04.189067243Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189618037Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189641964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} {"msg":"c69345e5-f150-4f5d-a8b6-98e5f7b81df5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189705285Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189723203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"0","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189758298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"0","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189784374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"1","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189809851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"1","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189836987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"2","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189866223Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"2","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.189904918Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.190209496Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.190247171Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"ds_connection connected from Ok(127.0.0.1:34331)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.190264778Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"ds_connection connected from Ok(127.0.0.1:47847)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.190280756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"ds_connection connected from Ok(127.0.0.1:35870)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.190364654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.190516863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"2","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.190681991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"1","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.191005736Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"0","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.191045521Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"2","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.19119814Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"1","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.191563629Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"0","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.191697981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.191718008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[0]R flush_numbers[0..12]: [213, 211, 214, 208, 183, 197, 214, 211, 207, 210, 203, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192166196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[0]R generation[0..12]: [118, 117, 119, 115, 100, 109, 119, 117, 114, 116, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192191343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192207221Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[1]R flush_numbers[0..12]: [213, 211, 214, 208, 183, 197, 214, 211, 207, 210, 203, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192222019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[1]R generation[0..12]: [118, 117, 119, 115, 100, 109, 119, 117, 114, 116, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192236427Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192250645Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[2]R flush_numbers[0..12]: [213, 211, 214, 208, 183, 197, 214, 211, 207, 210, 203, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192265583Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[2]R generation[0..12]: [118, 117, 119, 115, 100, 109, 119, 117, 114, 116, 112, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192280381Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192294799Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192308247Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:09:04.192321755Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192341882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192355341Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"":"downstairs","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192368809Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192384097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"0","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192398184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"1","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192413153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"client":"2","":"downstairs","session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192426971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192439569Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"c69345e5-f150-4f5d-a8b6-98e5f7b81df5 is now active with session: 4220b028-8416-4033-8b31-6b2fa43fa956","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192452757Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192465145Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672,"session_id":"4220b028-8416-4033-8b31-6b2fa43fa956"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192477574Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} {"msg":"Activated sub_volume c69345e5-f150-4f5d-a8b6-98e5f7b81df5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:04.192488992Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 28672 data: 4 5 4 5 6 7 5 02/30 Read block 402 len 16384 03/30 Write block 563 len 40960 data: 10 11 11 10 9 10 9 7 7 7 04/30 Read block 479 len 28672 05/30 Read block 588 len 20480 06/30 Read block 251 len 32768 07/30 Write block 483 len 36864 data: 11 14 13 11 10 9 9 10 9 08/30 Write block 164 len 8192 data: 12 12 09/30 Flush 10/30 Write block 257 len 20480 data: 14 15 17 17 14 11/30 Write block 321 len 8192 data: 8 7 12/30 Read block 141 len 12288 13/30 Read block 433 len 8192 14/30 Read block 190 len 16384 15/30 Read block 285 len 28672 16/30 Write block 537 len 4096 data: 13 17/30 Write block 214 len 16384 data: 13 11 9 12 18/30 Flush 19/30 Write block 268 len 20480 data: 9 6 6 6 5 20/30 Read block 581 len 4096 21/30 Read block 221 len 12288 22/30 Flush 23/30 Write block 372 len 4096 data: 8 24/30 Write block 193 len 4096 data: 11 25/30 Flush 26/30 Write block 471 len 4096 data: 12 27/30 Write block 384 len 20480 data: 5 5 7 7 9 28/30 Read block 45 len 40960 29/30 Read block 6 len 4096 30/30 Read block 537 len 4096 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:04.212611764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 115 115 115 208 208 208 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 119 119 119 214 214 214 F F F 7 140-159 117 117 117 211 211 211 F F F 8 160-179 120 120 120 215 215 215 F F F 9 180-199 120 120 120 218 218 218 F F F 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 120 120 120 216 216 216 F F F 13 260-279 120 120 120 217 217 217 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 120 120 120 216 216 216 F F F 17 340-359 119 119 119 214 214 214 F F F 18 360-379 120 120 120 218 218 218 F F F 19 380-399 115 115 115 208 208 208 T T T 20 400-419 110 110 110 200 200 200 F F F 21 420-439 118 118 118 213 213 213 F F F 22 440-459 118 118 118 213 213 213 F F F 23 460-479 115 115 115 208 208 208 T T T 24 480-499 120 120 120 215 215 215 F F F 25 500-519 118 118 118 213 213 213 F F F 26 520-539 120 120 120 216 216 216 F F F 27 540-559 119 119 119 214 214 214 F F F 28 560-579 120 120 120 215 215 215 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 120, Max flush: 218 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 115 115 115 208 208 208 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 119 119 119 214 214 214 F F F 7 140-159 117 117 117 211 211 211 F F F 8 160-179 120 120 120 215 215 215 F F F 9 180-199 120 120 120 218 218 218 F F F 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 120 120 120 216 216 216 F F F 13 260-279 120 120 120 217 217 217 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 120 120 120 216 216 216 F F F 17 340-359 119 119 119 214 214 214 F F F 18 360-379 120 120 120 218 218 218 F F F 19 380-399 121 121 121 219 219 219 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 118 118 118 213 213 213 F F F 22 440-459 118 118 118 213 213 213 F F F 23 460-479 121 121 121 219 219 219 F F F 24 480-499 120 120 120 215 215 215 F F F 25 500-519 118 118 118 213 213 213 F F F 26 520-539 120 120 120 216 216 216 F F F 27 540-559 119 119 119 214 214 214 F F F 28 560-579 120 120 120 215 215 215 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 121, Max flush: 219 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.475312276Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.519732267Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.520472935Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.520511449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.520526187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.520541065Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} {"msg":"Crucible 836222a7-f65d-45f7-a262-a74e1d7b85a7 has session id: 1a8dbba3-d074-4c54-9192-f9a62be1dc04","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521209753Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Upstairs opts: Upstairs UUID: 836222a7-f65d-45f7-a262-a74e1d7b85a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521257137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Crucible stats registered with UUID: 836222a7-f65d-45f7-a262-a74e1d7b85a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521280504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:08.521300401Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521778895Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521805011Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} {"msg":"836222a7-f65d-45f7-a262-a74e1d7b85a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521876332Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521894609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"0","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521932814Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"0","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.52195923Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"1","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.521987517Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"1","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.522015723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.522044749Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.522072695Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.522089572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.52210444Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"ds_connection connected from Ok(127.0.0.1:58858)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.522124138Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:08.522152654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:08.52218074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"ds_connection connected from Ok(127.0.0.1:34593)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.522205327Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:08.52239859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.5225414Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.523010486Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"0","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.523074007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"1","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.523656387Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"1","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.52378287Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"0","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.5242873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:08.524343742Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.523637864Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"ds_connection connected from Ok(127.0.0.1:50239)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.523772525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.524933556Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.525513116Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[0]R flush_numbers[0..12]: [213, 215, 214, 208, 183, 197, 214, 211, 215, 218, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526132211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[0]R generation[0..12]: [118, 120, 119, 115, 100, 109, 119, 117, 120, 120, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526153878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526169855Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[1]R flush_numbers[0..12]: [213, 215, 214, 208, 183, 197, 214, 211, 215, 218, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526184544Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[1]R generation[0..12]: [118, 120, 119, 115, 100, 109, 119, 117, 120, 120, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526204361Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526219369Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[2]R flush_numbers[0..12]: [213, 215, 214, 208, 183, 197, 214, 211, 215, 218, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526233647Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[2]R generation[0..12]: [118, 120, 119, 115, 100, 109, 119, 117, 120, 120, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526248025Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526262623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526276311Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526290399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526304257Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526317835Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"":"downstairs","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526331823Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526344451Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"0","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-03-18T05:09:18.52635854Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"1","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526378017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"client":"2","":"downstairs","session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526391825Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526404553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"836222a7-f65d-45f7-a262-a74e1d7b85a7 is now active with session: 1a8dbba3-d074-4c54-9192-f9a62be1dc04","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526419901Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526432489Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680,"session_id":"1a8dbba3-d074-4c54-9192-f9a62be1dc04"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526445528Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} {"msg":"Activated sub_volume 836222a7-f65d-45f7-a262-a74e1d7b85a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:18.526457206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 513 len 8192 data: 14 16 02/30 Flush 03/30 Write block 450 len 12288 data: 16 17 14 04/30 Write block 333 len 40960 data: 8 8 9 12 11 13 12 12 10 13 05/30 Write block 444 len 32768 data: 12 13 14 17 19 18 17 18 06/30 Write block 277 len 4096 data: 7 07/30 Read block 550 len 24576 08/30 Read block 479 len 16384 09/30 Read block 268 len 32768 10/30 Read block 589 len 32768 11/30 Read block 408 len 12288 12/30 Read block 515 len 12288 13/30 Read block 276 len 16384 14/30 Write block 496 len 24576 data: 11 11 11 11 11 9 15/30 Write block 369 len 24576 data: 10 12 10 9 7 5 16/30 Flush 17/30 Write block 190 len 16384 data: 11 13 11 12 18/30 Read block 281 len 28672 19/30 Write block 68 len 8192 data: 8 10 20/30 Flush 21/30 Write block 345 len 40960 data: 14 12 12 12 11 10 11 12 12 12 22/30 Write block 555 len 24576 data: 10 9 8 8 8 8 23/30 Flush 24/30 Write block 240 len 4096 data: 14 25/30 Read block 272 len 24576 26/30 Read block 199 len 40960 27/30 Read block 315 len 12288 28/30 Read block 154 len 16384 29/30 Read block 89 len 12288 30/30 Write block 509 len 40960 data: 10 12 13 13 15 17 17 14 11 10 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:18.547504931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 122 122 122 222 222 222 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 119 119 119 214 214 214 F F F 7 140-159 117 117 117 211 211 211 F F F 8 160-179 120 120 120 215 215 215 F F F 9 180-199 122 122 122 222 222 222 F F F 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 120 120 120 216 216 216 T T T 13 260-279 122 122 122 221 221 221 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 122 122 122 221 221 221 F F F 17 340-359 122 122 122 223 223 223 F F F 18 360-379 122 122 122 221 221 221 F F F 19 380-399 121 121 121 219 219 219 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 118 118 118 213 213 213 F F F 22 440-459 122 122 122 221 221 221 F F F 23 460-479 121 121 121 219 219 219 F F F 24 480-499 122 122 122 221 221 221 F F F 25 500-519 122 122 122 221 221 221 T T T 26 520-539 120 120 120 216 216 216 F F F 27 540-559 122 122 122 223 223 223 F F F 28 560-579 122 122 122 223 223 223 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 122, Max flush: 223 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 122 122 122 222 222 222 F F F 4 080-099 100 100 100 183 183 183 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 119 119 119 214 214 214 F F F 7 140-159 117 117 117 211 211 211 F F F 8 160-179 120 120 120 215 215 215 F F F 9 180-199 122 122 122 222 222 222 F F F 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 123 123 123 224 224 224 F F F 13 260-279 122 122 122 221 221 221 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 122 122 122 221 221 221 F F F 17 340-359 122 122 122 223 223 223 F F F 18 360-379 122 122 122 221 221 221 F F F 19 380-399 121 121 121 219 219 219 F F F 20 400-419 110 110 110 200 200 200 F F F 21 420-439 118 118 118 213 213 213 F F F 22 440-459 122 122 122 221 221 221 F F F 23 460-479 121 121 121 219 219 219 F F F 24 480-499 122 122 122 221 221 221 F F F 25 500-519 123 123 123 224 224 224 F F F 26 520-539 120 120 120 216 216 216 F F F 27 540-559 122 122 122 223 223 223 F F F 28 560-579 122 122 122 223 223 223 F F F 29 580-599 101 101 101 184 184 184 F F F Max gen: 123, Max flush: 224 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.780913162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.826274452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.827014471Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.827050386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.827069313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.827084291Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} {"msg":"Crucible b776e5cc-2533-4ba4-9160-075a1a087c29 has session id: 44504500-2a27-4d1b-abdf-139878f6759b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.827712915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Upstairs opts: Upstairs UUID: b776e5cc-2533-4ba4-9160-075a1a087c29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.827757599Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Crucible stats registered with UUID: b776e5cc-2533-4ba4-9160-075a1a087c29","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.827777276Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:22.827793884Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828238273Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.82825989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} {"msg":"b776e5cc-2533-4ba4-9160-075a1a087c29 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828333389Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828355146Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"0","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.82840137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"0","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828429166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"1","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828458922Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"1","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828489738Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828530702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828558838Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828584765Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828608962Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:22.828638678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:22.828668123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43396)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.82869583Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45573)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.828722626Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:22.828752632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.82891462Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.829369987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"0","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.829400143Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"1","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.829948857Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"1","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.830080939Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"0","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.830568312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:22.830660749Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.83004811Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58003)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.83026243Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.831249354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.831764943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[0]R flush_numbers[0..12]: [213, 215, 214, 222, 183, 197, 214, 211, 215, 222, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832359062Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[0]R generation[0..12]: [118, 120, 119, 122, 100, 109, 119, 117, 120, 122, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832383998Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832402596Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[1]R flush_numbers[0..12]: [213, 215, 214, 222, 183, 197, 214, 211, 215, 222, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832420633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[1]R generation[0..12]: [118, 120, 119, 122, 100, 109, 119, 117, 120, 122, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832439191Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832457338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[2]R flush_numbers[0..12]: [213, 215, 214, 222, 183, 197, 214, 211, 215, 222, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832475166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[2]R generation[0..12]: [118, 120, 119, 122, 100, 109, 119, 117, 120, 122, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832498153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.83251631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832533268Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832550485Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832567093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832583571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"":"downstairs","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832600628Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2025-03-18T05:09:32.832622696Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"0","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832640363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"1","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832657421Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"client":"2","":"downstairs","session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832674538Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832690336Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"b776e5cc-2533-4ba4-9160-075a1a087c29 is now active with session: 44504500-2a27-4d1b-abdf-139878f6759b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832706694Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832722142Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688,"session_id":"44504500-2a27-4d1b-abdf-139878f6759b"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.83273781Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} {"msg":"Activated sub_volume b776e5cc-2533-4ba4-9160-075a1a087c29","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:32.832752308Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 350 len 20480 data: 11 12 13 13 13 02/30 Write block 75 len 40960 data: 5 10 11 11 10 10 12 8 8 10 03/30 Write block 286 len 12288 data: 8 10 9 04/30 Write block 478 len 40960 data: 7 8 8 8 9 12 15 14 12 11 05/30 Read block 103 len 32768 06/30 Read block 22 len 28672 07/30 Read block 495 len 20480 08/30 Flush 09/30 Write block 591 len 36864 data: 10 12 12 14 11 9 8 7 5 10/30 Write block 517 len 24576 data: 12 11 8 11 11 10 11/30 Write block 530 len 8192 data: 15 15 12/30 Read block 208 len 12288 13/30 Read block 496 len 28672 14/30 Read block 512 len 36864 15/30 Read block 484 len 28672 16/30 Write block 563 len 36864 data: 11 12 12 11 10 11 10 8 8 17/30 Read block 446 len 20480 18/30 Write block 282 len 24576 data: 8 7 6 7 9 11 19/30 Read block 546 len 40960 20/30 Write block 268 len 16384 data: 10 7 7 7 21/30 Write block 128 len 32768 data: 12 10 9 10 10 12 11 14 22/30 Write block 251 len 36864 data: 7 6 7 8 12 13 15 16 18 23/30 Write block 327 len 28672 data: 6 7 8 10 11 9 9 24/30 Write block 366 len 36864 data: 8 9 9 11 13 11 10 8 6 25/30 Read block 562 len 8192 26/30 Write block 186 len 16384 data: 9 9 10 11 27/30 Read block 453 len 28672 28/30 Write block 413 len 36864 data: 11 11 11 9 8 9 9 7 9 29/30 Read block 231 len 20480 30/30 Read block 470 len 36864 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 9 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 21 21 22 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:32.855371947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 124 124 124 225 225 225 F F F 4 080-099 124 124 124 225 225 225 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 119 119 119 214 214 214 T T T 7 140-159 117 117 117 211 211 211 F F F 8 160-179 120 120 120 215 215 215 F F F 9 180-199 122 122 122 222 222 222 T T T 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 123 123 123 224 224 224 T T T 13 260-279 122 122 122 221 221 221 T T T 14 280-299 124 124 124 225 225 225 T T T 15 300-319 114 114 114 206 206 206 F F F 16 320-339 122 122 122 221 221 221 T T T 17 340-359 124 124 124 225 225 225 F F F 18 360-379 122 122 122 221 221 221 T T T 19 380-399 121 121 121 219 219 219 F F F 20 400-419 110 110 110 200 200 200 T T T 21 420-439 118 118 118 213 213 213 T T T 22 440-459 122 122 122 221 221 221 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 124 124 124 225 225 225 F F F 25 500-519 123 123 123 224 224 224 T T T 26 520-539 120 120 120 216 216 216 T T T 27 540-559 122 122 122 223 223 223 F F F 28 560-579 122 122 122 223 223 223 T T T 29 580-599 101 101 101 184 184 184 T T T Max gen: 124, Max flush: 225 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 124 124 124 225 225 225 F F F 4 080-099 124 124 124 225 225 225 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 117 117 117 211 211 211 F F F 8 160-179 120 120 120 215 215 215 F F F 9 180-199 125 125 125 226 226 226 F F F 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 125 125 125 226 226 226 F F F 13 260-279 125 125 125 226 226 226 F F F 14 280-299 125 125 125 226 226 226 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 125 125 125 226 226 226 F F F 17 340-359 124 124 124 225 225 225 F F F 18 360-379 125 125 125 226 226 226 F F F 19 380-399 121 121 121 219 219 219 F F F 20 400-419 125 125 125 226 226 226 F F F 21 420-439 125 125 125 226 226 226 F F F 22 440-459 122 122 122 221 221 221 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 124 124 124 225 225 225 F F F 25 500-519 125 125 125 226 226 226 F F F 26 520-539 125 125 125 226 226 226 F F F 27 540-559 122 122 122 223 223 223 F F F 28 560-579 125 125 125 226 226 226 F F F 29 580-599 125 125 125 226 226 226 F F F Max gen: 125, Max flush: 226 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.14885945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:47.192419249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.192821293Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.22476413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.225542343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.225578288Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.225592496Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.225607314Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} {"msg":"Crucible fe7b8ac8-8197-4dc8-aad6-ca19bc4520e4 has session id: 607223e7-8719-4c65-9407-e2702bd87694","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226224969Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Upstairs opts: Upstairs UUID: fe7b8ac8-8197-4dc8-aad6-ca19bc4520e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226258364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Crucible stats registered with UUID: fe7b8ac8-8197-4dc8-aad6-ca19bc4520e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226274302Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:47.226287411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226703463Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226736719Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} {"msg":"fe7b8ac8-8197-4dc8-aad6-ca19bc4520e4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.2267999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226833175Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"0","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226889608Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"0","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226933522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"1","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.226970737Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"1","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227006992Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"2","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227041847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"2","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227070203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227100409Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227130815Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"ds_connection connected from Ok(127.0.0.1:41694)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227159531Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"ds_connection connected from Ok(127.0.0.1:53400)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227222202Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"ds_connection connected from Ok(127.0.0.1:36087)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227347954Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227902908Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"2","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.227931374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"1","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.228082644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"0","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.228590224Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"1","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.228631388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"2","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.228659134Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"0","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229176233Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229280909Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[0]R flush_numbers[0..12]: [213, 215, 214, 225, 225, 197, 226, 211, 215, 226, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.22934603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[0]R generation[0..12]: [118, 120, 119, 124, 124, 109, 125, 117, 120, 125, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229374256Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229391104Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[1]R flush_numbers[0..12]: [213, 215, 214, 225, 225, 197, 226, 211, 215, 226, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229406122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[1]R generation[0..12]: [118, 120, 119, 124, 124, 109, 125, 117, 120, 125, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229420889Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229435777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[2]R flush_numbers[0..12]: [213, 215, 214, 225, 225, 197, 226, 211, 215, 226, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229450335Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[2]R generation[0..12]: [118, 120, 119, 124, 124, 109, 125, 117, 120, 125, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229464654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229479092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.22949239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229510157Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229524135Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229537873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"":"downstairs","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229551391Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.22956373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"0","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229577578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"1","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229597755Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"client":"2","":"downstairs","session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229611623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229624172Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"fe7b8ac8-8197-4dc8-aad6-ca19bc4520e4 is now active with session: 607223e7-8719-4c65-9407-e2702bd87694","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.22963734Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229649578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696,"session_id":"607223e7-8719-4c65-9407-e2702bd87694"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229661976Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} {"msg":"Activated sub_volume fe7b8ac8-8197-4dc8-aad6-ca19bc4520e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:47.229673525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 548 len 32768 data: 9 9 10 11 11 10 10 11 02/30 Write block 449 len 4096 data: 19 03/30 Read block 98 len 40960 04/30 Read block 169 len 16384 05/30 Read block 70 len 40960 06/30 Read block 33 len 36864 07/30 Read block 581 len 28672 08/30 Read block 384 len 40960 09/30 Read block 398 len 24576 10/30 Read block 377 len 40960 11/30 Read block 489 len 8192 12/30 Read block 559 len 36864 13/30 Write block 575 len 24576 data: 7 5 7 10 11 10 14/30 Read block 178 len 32768 15/30 Read block 505 len 40960 16/30 Write block 82 len 20480 data: 9 9 11 9 12 17/30 Write block 585 len 12288 data: 8 8 8 18/30 Read block 409 len 36864 19/30 Write block 430 len 36864 data: 10 9 6 6 7 6 5 4 3 20/30 Write block 147 len 8192 data: 11 11 21/30 Flush 22/30 Write block 418 len 20480 data: 10 10 8 10 12 23/30 Write block 161 len 4096 data: 10 24/30 Read block 437 len 12288 25/30 Read block 93 len 24576 26/30 Write block 149 len 24576 data: 12 12 11 11 10 13 27/30 Write block 65 len 4096 data: 13 28/30 Write block 265 len 36864 data: 13 13 11 11 8 8 8 6 7 29/30 Read block 496 len 40960 30/30 Read block 68 len 24576 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:47.251768856Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 124 124 124 225 225 225 T T T 4 080-099 126 126 126 227 227 227 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 126 126 126 227 227 227 T T T 8 160-179 120 120 120 215 215 215 T T T 9 180-199 125 125 125 226 226 226 F F F 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 125 125 125 226 226 226 F F F 13 260-279 125 125 125 226 226 226 T T T 14 280-299 125 125 125 226 226 226 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 125 125 125 226 226 226 F F F 17 340-359 124 124 124 225 225 225 F F F 18 360-379 125 125 125 226 226 226 F F F 19 380-399 121 121 121 219 219 219 F F F 20 400-419 125 125 125 226 226 226 T T T 21 420-439 126 126 126 227 227 227 T T T 22 440-459 126 126 126 227 227 227 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 124 124 124 225 225 225 F F F 25 500-519 125 125 125 226 226 226 F F F 26 520-539 125 125 125 226 226 226 F F F 27 540-559 126 126 126 227 227 227 F F F 28 560-579 126 126 126 227 227 227 F F F 29 580-599 126 126 126 227 227 227 F F F Max gen: 126, Max flush: 227 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 119 119 119 214 214 214 F F F 3 060-079 127 127 127 228 228 228 F F F 4 080-099 126 126 126 227 227 227 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 127 127 127 228 228 228 F F F 8 160-179 127 127 127 228 228 228 F F F 9 180-199 125 125 125 226 226 226 F F F 10 200-219 120 120 120 216 216 216 F F F 11 220-239 118 118 118 212 212 212 F F F 12 240-259 125 125 125 226 226 226 F F F 13 260-279 127 127 127 228 228 228 F F F 14 280-299 125 125 125 226 226 226 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 125 125 125 226 226 226 F F F 17 340-359 124 124 124 225 225 225 F F F 18 360-379 125 125 125 226 226 226 F F F 19 380-399 121 121 121 219 219 219 F F F 20 400-419 127 127 127 228 228 228 F F F 21 420-439 127 127 127 228 228 228 F F F 22 440-459 126 126 126 227 227 227 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 124 124 124 225 225 225 F F F 25 500-519 125 125 125 226 226 226 F F F 26 520-539 125 125 125 226 226 226 F F F 27 540-559 126 126 126 227 227 227 F F F 28 560-579 126 126 126 227 227 227 F F F 29 580-599 126 126 126 227 227 227 F F F Max gen: 127, Max flush: 228 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.541392586Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:51.587175298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.587658052Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.619409275Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.620184858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.620220983Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.620239161Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.620257478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} {"msg":"Crucible b727a37d-154d-4277-bb8f-05ea47dac16c has session id: f389ef3e-592c-4a0c-b669-53e3f09373a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.620968411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Upstairs opts: Upstairs UUID: b727a37d-154d-4277-bb8f-05ea47dac16c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621034461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Crucible stats registered with UUID: b727a37d-154d-4277-bb8f-05ea47dac16c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621063017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:51.621080465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621491379Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621513026Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} {"msg":"b727a37d-154d-4277-bb8f-05ea47dac16c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621589785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621608143Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"0","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621637109Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"0","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621652656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"1","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621667454Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"1","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621681123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"2","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621695661Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"2","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621709149Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621723857Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621738115Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48875)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.62184382Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63601)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.621884915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:60174)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.622013237Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.622661578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"1","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.622689984Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"2","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.622806498Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"0","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.623386098Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"1","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.623448869Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"0","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.623477096Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"2","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624107709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624133425Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[0]R flush_numbers[0..12]: [213, 215, 214, 228, 227, 197, 226, 228, 228, 226, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624176049Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[0]R generation[0..12]: [118, 120, 119, 127, 126, 109, 125, 127, 127, 125, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624194607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624210025Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[1]R flush_numbers[0..12]: [213, 215, 214, 228, 227, 197, 226, 228, 228, 226, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624228092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[1]R generation[0..12]: [118, 120, 119, 127, 126, 109, 125, 127, 127, 125, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624250489Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624276735Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[2]R flush_numbers[0..12]: [213, 215, 214, 228, 227, 197, 226, 228, 228, 226, 216, 212]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624304602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[2]R generation[0..12]: [118, 120, 119, 127, 126, 109, 125, 127, 127, 125, 120, 118]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624331288Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624359174Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.62439095Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624411937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624426215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624439853Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"":"downstairs","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.62446148Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624485497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"0","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624510973Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"1","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624526431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"client":"2","":"downstairs","session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624542899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624562266Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"b727a37d-154d-4277-bb8f-05ea47dac16c is now active with session: f389ef3e-592c-4a0c-b669-53e3f09373a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624586023Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.62460999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704,"session_id":"f389ef3e-592c-4a0c-b669-53e3f09373a8"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624629127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} {"msg":"Activated sub_volume b727a37d-154d-4277-bb8f-05ea47dac16c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:51.624641436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 16384 data: 6 6 5 7 02/30 Read block 181 len 28672 03/30 Read block 29 len 40960 04/30 Flush 05/30 Write block 55 len 12288 data: 15 13 13 06/30 Flush 07/30 Write block 329 len 24576 data: 9 11 12 10 10 9 08/30 Write block 45 len 12288 data: 15 15 14 09/30 Read block 400 len 24576 10/30 Write block 434 len 32768 data: 8 7 6 5 4 4 6 9 11/30 Flush 12/30 Write block 44 len 4096 data: 18 13/30 Write block 88 len 36864 data: 12 12 14 15 16 15 14 12 13 14/30 Write block 530 len 4096 data: 16 15/30 Read block 459 len 40960 16/30 Write block 158 len 4096 data: 13 17/30 Write block 559 len 40960 data: 9 9 10 10 12 13 13 12 11 12 18/30 Write block 281 len 40960 data: 9 9 8 7 8 10 12 10 9 9 19/30 Read block 61 len 36864 20/30 Write block 237 len 12288 data: 14 12 13 21/30 Read block 205 len 40960 22/30 Write block 324 len 28672 data: 8 9 7 7 8 10 12 23/30 Read block 57 len 40960 24/30 Read block 71 len 24576 25/30 Write block 484 len 32768 data: 16 15 13 12 10 10 11 10 26/30 Read block 171 len 40960 27/30 Read block 380 len 36864 28/30 Write block 208 len 12288 data: 10 11 11 29/30 Write block 143 len 28672 data: 8 8 11 11 12 12 13 30/30 Read block 102 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 1 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:51.645469971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 128 128 128 231 231 231 T T T 3 060-079 127 127 127 228 228 228 F F F 4 080-099 126 126 126 227 227 227 T T T 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 127 127 127 228 228 228 T T T 8 160-179 127 127 127 228 228 228 F F F 9 180-199 125 125 125 226 226 226 F F F 10 200-219 120 120 120 216 216 216 T T T 11 220-239 118 118 118 212 212 212 T T T 12 240-259 125 125 125 226 226 226 F F F 13 260-279 127 127 127 228 228 228 F F F 14 280-299 125 125 125 226 226 226 T T T 15 300-319 114 114 114 206 206 206 F F F 16 320-339 128 128 128 231 231 231 T T T 17 340-359 124 124 124 225 225 225 F F F 18 360-379 125 125 125 226 226 226 F F F 19 380-399 128 128 128 229 229 229 F F F 20 400-419 127 127 127 228 228 228 F F F 21 420-439 128 128 128 231 231 231 F F F 22 440-459 128 128 128 231 231 231 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 124 124 124 225 225 225 T T T 25 500-519 125 125 125 226 226 226 F F F 26 520-539 125 125 125 226 226 226 T T T 27 540-559 126 126 126 227 227 227 T T T 28 560-579 126 126 126 227 227 227 T T T 29 580-599 126 126 126 227 227 227 F F F Max gen: 128, Max flush: 231 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 120 120 120 215 215 215 F F F 2 040-059 129 129 129 232 232 232 F F F 3 060-079 127 127 127 228 228 228 F F F 4 080-099 129 129 129 232 232 232 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 129 129 129 232 232 232 F F F 8 160-179 127 127 127 228 228 228 F F F 9 180-199 125 125 125 226 226 226 F F F 10 200-219 129 129 129 232 232 232 F F F 11 220-239 129 129 129 232 232 232 F F F 12 240-259 125 125 125 226 226 226 F F F 13 260-279 127 127 127 228 228 228 F F F 14 280-299 129 129 129 232 232 232 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 129 129 129 232 232 232 F F F 17 340-359 124 124 124 225 225 225 F F F 18 360-379 125 125 125 226 226 226 F F F 19 380-399 128 128 128 229 229 229 F F F 20 400-419 127 127 127 228 228 228 F F F 21 420-439 128 128 128 231 231 231 F F F 22 440-459 128 128 128 231 231 231 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 129 129 129 232 232 232 F F F 25 500-519 125 125 125 226 226 226 F F F 26 520-539 129 129 129 232 232 232 F F F 27 540-559 129 129 129 232 232 232 F F F 28 560-579 129 129 129 232 232 232 F F F 29 580-599 126 126 126 227 227 227 F F F Max gen: 129, Max flush: 232 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.953353659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:55.996759859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:55.997277758Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.029388941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.030164394Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.030212868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.030227206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.030241954Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} {"msg":"Crucible 0561d34b-569a-43e0-8677-bba5ef9990ef has session id: 807dc37d-de25-4984-a39b-4e8fad69f649","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.03085222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Upstairs opts: Upstairs UUID: 0561d34b-569a-43e0-8677-bba5ef9990ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.030901953Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Crucible stats registered with UUID: 0561d34b-569a-43e0-8677-bba5ef9990ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.03092552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:56.030939648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031439769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031474184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} {"msg":"0561d34b-569a-43e0-8677-bba5ef9990ef active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031553323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031572821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"0","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031602957Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"0","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031626363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"1","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031661269Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"1","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031694884Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"2","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031729659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"2","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031760275Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031788181Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031822926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"ds_connection connected from Ok(127.0.0.1:57181)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.031960137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"ds_connection connected from Ok(127.0.0.1:58097)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.032132604Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"ds_connection connected from Ok(127.0.0.1:37724)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.032177338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.032681158Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"1","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.032731081Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"2","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.032808731Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"0","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.033337878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"1","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.033504175Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"0","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.03353891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"2","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.033953463Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034072147Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[0]R flush_numbers[0..12]: [213, 215, 232, 228, 232, 197, 226, 232, 228, 226, 232, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034201909Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[0]R generation[0..12]: [118, 120, 129, 127, 129, 109, 125, 129, 127, 125, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034223156Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034238424Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[1]R flush_numbers[0..12]: [213, 215, 232, 228, 232, 197, 226, 232, 228, 226, 232, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034252982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[1]R generation[0..12]: [118, 120, 129, 127, 129, 109, 125, 129, 127, 125, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.0342674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034281828Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[2]R flush_numbers[0..12]: [213, 215, 232, 228, 232, 197, 226, 232, 228, 226, 232, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034295926Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[2]R generation[0..12]: [118, 120, 129, 127, 129, 109, 125, 129, 127, 125, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034314364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034329742Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.03434326Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034357268Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:09:56.034370806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034388823Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"":"downstairs","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034402552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.03441497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"0","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034428858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"1","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034442716Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"client":"2","":"downstairs","session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034456534Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034469232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"0561d34b-569a-43e0-8677-bba5ef9990ef is now active with session: 807dc37d-de25-4984-a39b-4e8fad69f649","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.03448251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034495039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712,"session_id":"807dc37d-de25-4984-a39b-4e8fad69f649"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034507627Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} {"msg":"Activated sub_volume 0561d34b-569a-43e0-8677-bba5ef9990ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:56.034519255Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 589 len 20480 data: 11 13 11 13 13 02/30 Read block 58 len 12288 03/30 Write block 22 len 28672 data: 6 7 7 6 6 5 5 04/30 Read block 473 len 8192 05/30 Read block 191 len 12288 06/30 Write block 191 len 36864 data: 14 12 13 10 9 10 8 9 9 07/30 Write block 539 len 16384 data: 14 11 9 8 08/30 Read block 308 len 4096 09/30 Read block 240 len 12288 10/30 Read block 354 len 36864 11/30 Write block 45 len 40960 data: 16 16 15 13 8 9 11 10 12 12 12/30 Write block 398 len 24576 data: 8 7 8 8 9 12 13/30 Write block 335 len 28672 data: 10 13 12 14 13 13 11 14/30 Write block 435 len 20480 data: 8 7 6 5 5 15/30 Flush 16/30 Write block 276 len 20480 data: 7 8 8 8 10 17/30 Read block 162 len 40960 18/30 Flush 19/30 Write block 502 len 20480 data: 10 13 12 10 11 20/30 Read block 342 len 4096 21/30 Read block 187 len 28672 22/30 Write block 571 len 8192 data: 9 8 23/30 Write block 49 len 24576 data: 9 10 12 11 13 13 24/30 Flush 25/30 Write block 196 len 4096 data: 11 26/30 Flush 27/30 Write block 409 len 28672 data: 11 11 9 10 12 12 12 28/30 Read block 492 len 12288 29/30 Write block 512 len 4096 data: 14 30/30 Write block 371 len 12288 data: 12 11 9 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 3 2 3 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:56.054646777Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 130 130 130 235 235 235 F F F 3 060-079 127 127 127 228 228 228 F F F 4 080-099 129 129 129 232 232 232 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 129 129 129 232 232 232 F F F 8 160-179 127 127 127 228 228 228 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 129 129 129 232 232 232 F F F 11 220-239 129 129 129 232 232 232 F F F 12 240-259 125 125 125 226 226 226 F F F 13 260-279 130 130 130 234 234 234 F F F 14 280-299 130 130 130 234 234 234 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 130 130 130 233 233 233 F F F 17 340-359 130 130 130 233 233 233 F F F 18 360-379 125 125 125 226 226 226 T T T 19 380-399 130 130 130 233 233 233 F F F 20 400-419 130 130 130 233 233 233 T T T 21 420-439 130 130 130 233 233 233 F F F 22 440-459 128 128 128 231 231 231 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 129 129 129 232 232 232 F F F 25 500-519 130 130 130 235 235 235 T T T 26 520-539 130 130 130 233 233 233 F F F 27 540-559 130 130 130 233 233 233 F F F 28 560-579 130 130 130 235 235 235 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 130, Max flush: 236 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 130 130 130 235 235 235 F F F 3 060-079 127 127 127 228 228 228 F F F 4 080-099 129 129 129 232 232 232 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 129 129 129 232 232 232 F F F 8 160-179 127 127 127 228 228 228 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 129 129 129 232 232 232 F F F 11 220-239 129 129 129 232 232 232 F F F 12 240-259 125 125 125 226 226 226 F F F 13 260-279 130 130 130 234 234 234 F F F 14 280-299 130 130 130 234 234 234 F F F 15 300-319 114 114 114 206 206 206 F F F 16 320-339 130 130 130 233 233 233 F F F 17 340-359 130 130 130 233 233 233 F F F 18 360-379 131 131 131 237 237 237 F F F 19 380-399 130 130 130 233 233 233 F F F 20 400-419 131 131 131 237 237 237 F F F 21 420-439 130 130 130 233 233 233 F F F 22 440-459 128 128 128 231 231 231 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 129 129 129 232 232 232 F F F 25 500-519 131 131 131 237 237 237 F F F 26 520-539 130 130 130 233 233 233 F F F 27 540-559 130 130 130 233 233 233 F F F 28 560-579 130 130 130 235 235 235 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 131, Max flush: 237 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.320786736Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.366529764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.367312327Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.367355581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.367370699Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.367385997Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} {"msg":"Crucible b270b1eb-7af5-4897-9094-b085ef6060e6 has session id: 4dbec1b8-f57a-4c2c-b06d-524612edcb6c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.3680132Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Upstairs opts: Upstairs UUID: b270b1eb-7af5-4897-9094-b085ef6060e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368047496Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Crucible stats registered with UUID: b270b1eb-7af5-4897-9094-b085ef6060e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368063123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:00.368076182Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368630845Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368658072Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} {"msg":"b270b1eb-7af5-4897-9094-b085ef6060e6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.36874177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368768566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"0","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368803482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"0","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.36881911Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"1","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368839427Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"1","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368866353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"2","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368896029Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"2","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368922275Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368949452Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.368984117Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61715)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.36925156Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44035)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.369302693Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33874)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.369337488Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.370083076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"0","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.370112162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"2","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.370293047Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"1","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.370794388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"0","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.370823244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"2","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.370839941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"1","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371376858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.37150333Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[0]R flush_numbers[0..12]: [213, 233, 235, 228, 232, 197, 226, 232, 228, 236, 232, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371566652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[0]R generation[0..12]: [118, 130, 130, 127, 129, 109, 125, 129, 127, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371595268Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371615475Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[1]R flush_numbers[0..12]: [213, 233, 235, 228, 232, 197, 226, 232, 228, 236, 232, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371630723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[1]R generation[0..12]: [118, 130, 130, 127, 129, 109, 125, 129, 127, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371645631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371660319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[2]R flush_numbers[0..12]: [213, 233, 235, 228, 232, 197, 226, 232, 228, 236, 232, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371674877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[2]R generation[0..12]: [118, 130, 130, 127, 129, 109, 125, 129, 127, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371689135Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371703773Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371717371Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371731209Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371751286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371764684Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"":"downstairs","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371778472Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371790931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"0","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371805029Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"1","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371820387Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"client":"2","":"downstairs","session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371834315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371847073Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"b270b1eb-7af5-4897-9094-b085ef6060e6 is now active with session: 4dbec1b8-f57a-4c2c-b06d-524612edcb6c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371860201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371872499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720,"session_id":"4dbec1b8-f57a-4c2c-b06d-524612edcb6c"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371884888Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} {"msg":"Activated sub_volume b270b1eb-7af5-4897-9094-b085ef6060e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:00.371896356Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 198 len 28672 data: 10 10 9 8 8 7 7 02/30 Read block 344 len 4096 03/30 Read block 203 len 40960 04/30 Flush 05/30 Write block 494 len 40960 data: 10 12 12 12 12 12 12 10 11 14 06/30 Read block 460 len 28672 07/30 Write block 212 len 4096 data: 9 08/30 Read block 345 len 36864 09/30 Write block 279 len 20480 data: 9 11 10 10 9 10/30 Read block 528 len 24576 11/30 Write block 310 len 12288 data: 6 9 10 12/30 Read block 551 len 40960 13/30 Write block 174 len 36864 data: 11 10 8 8 8 11 13 14 12 14/30 Write block 435 len 32768 data: 9 8 7 6 6 7 10 9 15/30 Read block 486 len 16384 16/30 Write block 209 len 36864 data: 12 12 9 10 13 14 12 10 13 17/30 Write block 527 len 28672 data: 11 13 15 17 16 15 15 18/30 Write block 480 len 36864 data: 9 9 10 13 17 16 14 13 11 19/30 Write block 541 len 40960 data: 10 9 8 6 8 6 8 10 10 11 20/30 Flush 21/30 Write block 241 len 40960 data: 14 11 12 12 13 13 13 14 12 11 22/30 Write block 260 len 36864 data: 18 15 16 14 14 14 14 12 12 23/30 Read block 43 len 8192 24/30 Read block 8 len 32768 25/30 Write block 518 len 36864 data: 12 9 12 12 11 8 9 10 13 26/30 Read block 377 len 32768 27/30 Write block 380 len 24576 data: 7 8 8 7 6 6 28/30 Write block 362 len 32768 data: 5 5 6 8 9 10 10 12 29/30 Write block 333 len 40960 data: 11 10 11 14 13 15 14 14 12 14 30/30 Write block 497 len 28672 data: 13 13 13 13 11 12 15 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:00.393472879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 130 130 130 235 235 235 F F F 3 060-079 127 127 127 228 228 228 F F F 4 080-099 129 129 129 232 232 232 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 129 129 129 232 232 232 F F F 8 160-179 132 132 132 239 239 239 F F F 9 180-199 132 132 132 239 239 239 F F F 10 200-219 132 132 132 239 239 239 F F F 11 220-239 129 129 129 232 232 232 F F F 12 240-259 125 125 125 226 226 226 T T T 13 260-279 132 132 132 239 239 239 T T T 14 280-299 132 132 132 239 239 239 F F F 15 300-319 132 132 132 239 239 239 F F F 16 320-339 130 130 130 233 233 233 T T T 17 340-359 130 130 130 233 233 233 T T T 18 360-379 131 131 131 237 237 237 T T T 19 380-399 130 130 130 233 233 233 T T T 20 400-419 131 131 131 237 237 237 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 132 132 132 239 239 239 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 132 132 132 239 239 239 T T T 25 500-519 132 132 132 239 239 239 T T T 26 520-539 132 132 132 239 239 239 T T T 27 540-559 132 132 132 239 239 239 F F F 28 560-579 130 130 130 235 235 235 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 132, Max flush: 239 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 130 130 130 235 235 235 F F F 3 060-079 127 127 127 228 228 228 F F F 4 080-099 129 129 129 232 232 232 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 125 125 125 226 226 226 F F F 7 140-159 129 129 129 232 232 232 F F F 8 160-179 132 132 132 239 239 239 F F F 9 180-199 132 132 132 239 239 239 F F F 10 200-219 132 132 132 239 239 239 F F F 11 220-239 129 129 129 232 232 232 F F F 12 240-259 133 133 133 240 240 240 F F F 13 260-279 133 133 133 240 240 240 F F F 14 280-299 132 132 132 239 239 239 F F F 15 300-319 132 132 132 239 239 239 F F F 16 320-339 133 133 133 240 240 240 F F F 17 340-359 133 133 133 240 240 240 F F F 18 360-379 133 133 133 240 240 240 F F F 19 380-399 133 133 133 240 240 240 F F F 20 400-419 131 131 131 237 237 237 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 132 132 132 239 239 239 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 133 133 133 240 240 240 F F F 26 520-539 133 133 133 240 240 240 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 130 130 130 235 235 235 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 133, Max flush: 240 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.657823988Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.702317099Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.70310588Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.703142155Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.703156233Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.703170671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} {"msg":"Crucible 99923793-0d82-4dec-be9d-305fd23a9f96 has session id: 044e6770-74e6-4a98-bd90-4d389d06a520","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.703880894Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Upstairs opts: Upstairs UUID: 99923793-0d82-4dec-be9d-305fd23a9f96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.703937656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Crucible stats registered with UUID: 99923793-0d82-4dec-be9d-305fd23a9f96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.703968682Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:14.703995348Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704478522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704520376Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} {"msg":"99923793-0d82-4dec-be9d-305fd23a9f96 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704579538Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704619832Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"0","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704668585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"0","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704702021Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"1","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704774971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"1","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704804267Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"2","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704831003Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"2","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704857599Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.704884036Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.705004779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"ds_connection connected from Ok(127.0.0.1:59001)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.705044334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"ds_connection connected from Ok(127.0.0.1:37005)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.705066131Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"ds_connection connected from Ok(127.0.0.1:63499)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.705085678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.705589349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"2","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.705618355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"0","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.706230061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"0","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.706269715Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"2","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.707023032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.707049678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.707304943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"1","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.707814473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"1","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[0]R flush_numbers[0..12]: [213, 233, 235, 228, 232, 197, 226, 232, 239, 239, 239, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708327022Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[0]R generation[0..12]: [118, 130, 130, 127, 129, 109, 125, 129, 132, 132, 132, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708352339Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708368187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[1]R flush_numbers[0..12]: [213, 233, 235, 228, 232, 197, 226, 232, 239, 239, 239, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708383354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[1]R generation[0..12]: [118, 130, 130, 127, 129, 109, 125, 129, 132, 132, 132, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708398072Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.70841253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[2]R flush_numbers[0..12]: [213, 233, 235, 228, 232, 197, 226, 232, 239, 239, 239, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708426899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[2]R generation[0..12]: [118, 130, 130, 127, 129, 109, 125, 129, 132, 132, 132, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708440837Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708458464Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708472032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:10:14.70848569Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708504778Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708518266Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"":"downstairs","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708532124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708545232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"0","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.70855925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"1","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708572739Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"client":"2","":"downstairs","session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708586406Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708598925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"99923793-0d82-4dec-be9d-305fd23a9f96 is now active with session: 044e6770-74e6-4a98-bd90-4d389d06a520","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708612083Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708624261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728,"session_id":"044e6770-74e6-4a98-bd90-4d389d06a520"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.70863669Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} {"msg":"Activated sub_volume 99923793-0d82-4dec-be9d-305fd23a9f96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:14.708648198Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 444 len 28672 data: 13 14 15 18 20 20 18 02/30 Read block 511 len 36864 03/30 Write block 122 len 16384 data: 15 13 15 17 04/30 Read block 229 len 4096 05/30 Write block 63 len 4096 data: 15 06/30 Read block 110 len 20480 07/30 Write block 412 len 28672 data: 11 13 13 13 10 9 11 08/30 Read block 223 len 24576 09/30 Write block 180 len 40960 data: 14 15 13 10 8 9 10 10 11 12 10/30 Read block 186 len 8192 11/30 Write block 122 len 8192 data: 16 14 12/30 Flush 13/30 Write block 333 len 32768 data: 12 11 12 15 14 16 15 15 14/30 Flush 15/30 Write block 375 len 24576 data: 5 6 4 5 6 8 16/30 Flush 17/30 Write block 279 len 20480 data: 10 12 11 11 10 18/30 Read block 402 len 12288 19/30 Flush 20/30 Write block 154 len 32768 data: 14 14 16 15 14 11 11 11 21/30 Write block 237 len 36864 data: 15 13 14 15 15 12 13 13 14 22/30 Write block 345 len 16384 data: 15 13 13 13 23/30 Read block 472 len 20480 24/30 Write block 379 len 16384 data: 7 9 9 9 25/30 Read block 100 len 12288 26/30 Write block 212 len 20480 data: 11 14 15 13 11 27/30 Write block 241 len 20480 data: 16 13 14 14 15 28/30 Write block 570 len 16384 data: 9 10 9 6 29/30 Read block 234 len 28672 30/30 Write block 356 len 8192 data: 12 10 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 7 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 4 7 Done 10 9 7 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:14.728857998Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 130 130 130 235 235 235 F F F 3 060-079 134 134 134 241 241 241 F F F 4 080-099 129 129 129 232 232 232 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 134 134 134 241 241 241 F F F 7 140-159 129 129 129 232 232 232 T T T 8 160-179 132 132 132 239 239 239 T T T 9 180-199 134 134 134 241 241 241 F F F 10 200-219 132 132 132 239 239 239 T T T 11 220-239 129 129 129 232 232 232 T T T 12 240-259 133 133 133 240 240 240 T T T 13 260-279 134 134 134 244 244 244 F F F 14 280-299 134 134 134 244 244 244 F F F 15 300-319 132 132 132 239 239 239 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 134 134 134 242 242 242 T T T 18 360-379 134 134 134 243 243 243 T T T 19 380-399 134 134 134 243 243 243 T T T 20 400-419 134 134 134 241 241 241 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 134 134 134 241 241 241 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 133 133 133 240 240 240 F F F 26 520-539 133 133 133 240 240 240 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 130 130 130 235 235 235 T T T 29 580-599 130 130 130 233 233 233 F F F Max gen: 134, Max flush: 244 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 130 130 130 235 235 235 F F F 3 060-079 134 134 134 241 241 241 F F F 4 080-099 129 129 129 232 232 232 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 134 134 134 241 241 241 F F F 7 140-159 135 135 135 245 245 245 F F F 8 160-179 135 135 135 245 245 245 F F F 9 180-199 134 134 134 241 241 241 F F F 10 200-219 135 135 135 245 245 245 F F F 11 220-239 135 135 135 245 245 245 F F F 12 240-259 135 135 135 245 245 245 F F F 13 260-279 134 134 134 244 244 244 F F F 14 280-299 134 134 134 244 244 244 F F F 15 300-319 132 132 132 239 239 239 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 135 135 135 245 245 245 F F F 18 360-379 135 135 135 245 245 245 F F F 19 380-399 135 135 135 245 245 245 F F F 20 400-419 134 134 134 241 241 241 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 134 134 134 241 241 241 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 133 133 133 240 240 240 F F F 26 520-539 133 133 133 240 240 240 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 135 135 135 245 245 245 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 135, Max flush: 245 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:28.999510203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:29.042798129Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.043271384Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.07514222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.075851423Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.075886048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.075903905Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.075918963Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} {"msg":"Crucible a5ae508f-c3ac-41e5-9fbc-5cf10785b09c has session id: d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.076584532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Upstairs opts: Upstairs UUID: a5ae508f-c3ac-41e5-9fbc-5cf10785b09c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.076619787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Crucible stats registered with UUID: a5ae508f-c3ac-41e5-9fbc-5cf10785b09c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.076635375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:29.076648583Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077021182Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077046028Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} {"msg":"a5ae508f-c3ac-41e5-9fbc-5cf10785b09c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077111229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077128747Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"0","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077156783Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"0","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077172421Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"1","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077192908Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"1","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077219674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"2","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.07724752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"2","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077278296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077306412Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.07732237Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60312)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077337688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36257)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077552229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57985)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.077606121Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.078040291Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"1","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.078078706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"2","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.07826846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"0","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.078780389Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"2","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.078872687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"1","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079073329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"0","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079444858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079479773Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[0]R flush_numbers[0..12]: [213, 233, 235, 241, 232, 197, 241, 245, 245, 241, 245, 245]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079728959Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[0]R generation[0..12]: [118, 130, 130, 134, 129, 109, 134, 135, 135, 134, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079769493Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.07979713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[1]R flush_numbers[0..12]: [213, 233, 235, 241, 232, 197, 241, 245, 245, 241, 245, 245]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079824816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[1]R generation[0..12]: [118, 130, 130, 134, 129, 109, 134, 135, 135, 134, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079850373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079877859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[2]R flush_numbers[0..12]: [213, 233, 235, 241, 232, 197, 241, 245, 245, 241, 245, 245]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079906715Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[2]R generation[0..12]: [118, 130, 130, 134, 129, 109, 134, 135, 135, 134, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079935241Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.079964857Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.08001231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080041336Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080066983Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080093219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"":"downstairs","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080120215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080144632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"0","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:10:29.080174388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"1","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080201344Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"client":"2","":"downstairs","session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080226271Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080240599Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"a5ae508f-c3ac-41e5-9fbc-5cf10785b09c is now active with session: d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080254107Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080266865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736,"session_id":"d04c98ae-eb6e-4e7e-8485-fa1eb2e40fe6"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080279533Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} {"msg":"Activated sub_volume a5ae508f-c3ac-41e5-9fbc-5cf10785b09c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:29.080291052Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 190 len 28672 data: 12 15 13 14 11 10 12 02/30 Write block 44 len 28672 data: 19 17 17 16 14 10 11 03/30 Read block 526 len 4096 04/30 Write block 376 len 32768 data: 7 5 6 8 10 10 10 8 05/30 Write block 574 len 12288 data: 6 8 6 06/30 Write block 62 len 28672 data: 14 16 13 14 10 10 9 07/30 Read block 137 len 20480 08/30 Write block 512 len 40960 data: 15 16 18 18 15 13 13 10 13 13 09/30 Read block 441 len 4096 10/30 Write block 300 len 16384 data: 5 5 5 6 11/30 Read block 339 len 20480 12/30 Read block 528 len 12288 13/30 Write block 389 len 12288 data: 6 8 7 14/30 Write block 237 len 28672 data: 16 14 15 16 17 14 15 15/30 Write block 222 len 24576 data: 11 10 10 10 12 13 16/30 Write block 86 len 16384 data: 13 11 13 13 17/30 Flush 18/30 Write block 83 len 16384 data: 10 12 10 14 19/30 Write block 250 len 4096 data: 12 20/30 Read block 85 len 20480 21/30 Read block 153 len 8192 22/30 Read block 400 len 36864 23/30 Write block 141 len 4096 data: 6 24/30 Write block 155 len 28672 data: 15 17 16 15 12 12 12 25/30 Write block 100 len 32768 data: 10 9 7 11 13 12 9 11 26/30 Read block 310 len 24576 27/30 Write block 102 len 32768 data: 8 12 14 13 10 12 12 12 28/30 Read block 232 len 4096 29/30 Read block 187 len 4096 30/30 Read block 116 len 24576 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 12 13 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:29.101456501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 136 136 136 246 246 246 F F F 3 060-079 136 136 136 246 246 246 F F F 4 080-099 136 136 136 246 246 246 T T T 5 100-119 109 109 109 197 197 197 T T T 6 120-139 134 134 134 241 241 241 F F F 7 140-159 135 135 135 245 245 245 T T T 8 160-179 135 135 135 245 245 245 T T T 9 180-199 136 136 136 246 246 246 F F F 10 200-219 135 135 135 245 245 245 F F F 11 220-239 136 136 136 246 246 246 F F F 12 240-259 136 136 136 246 246 246 T T T 13 260-279 134 134 134 244 244 244 F F F 14 280-299 134 134 134 244 244 244 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 135 135 135 245 245 245 F F F 18 360-379 136 136 136 246 246 246 F F F 19 380-399 136 136 136 246 246 246 F F F 20 400-419 134 134 134 241 241 241 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 134 134 134 241 241 241 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 136 136 136 246 246 246 F F F 26 520-539 136 136 136 246 246 246 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 136 136 136 246 246 246 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 136, Max flush: 246 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 136 136 136 246 246 246 F F F 3 060-079 136 136 136 246 246 246 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 134 134 134 241 241 241 F F F 7 140-159 137 137 137 247 247 247 F F F 8 160-179 137 137 137 247 247 247 F F F 9 180-199 136 136 136 246 246 246 F F F 10 200-219 135 135 135 245 245 245 F F F 11 220-239 136 136 136 246 246 246 F F F 12 240-259 137 137 137 247 247 247 F F F 13 260-279 134 134 134 244 244 244 F F F 14 280-299 134 134 134 244 244 244 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 135 135 135 245 245 245 F F F 18 360-379 136 136 136 246 246 246 F F F 19 380-399 136 136 136 246 246 246 F F F 20 400-419 134 134 134 241 241 241 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 134 134 134 241 241 241 F F F 23 460-479 124 124 124 225 225 225 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 136 136 136 246 246 246 F F F 26 520-539 136 136 136 246 246 246 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 136 136 136 246 246 246 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 137, Max flush: 247 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.376754231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.421074685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.421780358Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.421824672Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.42184217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.421860677Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} {"msg":"Crucible 87927870-13a1-4ce0-8754-33ad6d85bd36 has session id: 1d6fd0e4-4aca-4dc8-a068-d2349b2f145b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.42242299Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Upstairs opts: Upstairs UUID: 87927870-13a1-4ce0-8754-33ad6d85bd36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.422467253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Crucible stats registered with UUID: 87927870-13a1-4ce0-8754-33ad6d85bd36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.42248965Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:33.422509328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.422978373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423011429Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} {"msg":"87927870-13a1-4ce0-8754-33ad6d85bd36 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423066431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423092907Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"0","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423131232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"0","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423159968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423198743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423228479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"2","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423257745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"2","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423282581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423308118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423331434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59608)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423354182Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37177)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423379508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:33.423476894Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:33.423517879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:33.423567012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.423769294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.424064154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"2","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.424101999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"0","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.424574803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"2","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.42467588Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"0","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.425205657Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:33.42532958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.424452736Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55263)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.424542343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.42572625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.426594861Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[0]R flush_numbers[0..12]: [213, 233, 246, 246, 247, 247, 241, 247, 247, 246, 245, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427239472Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[0]R generation[0..12]: [118, 130, 136, 136, 137, 137, 134, 137, 137, 136, 135, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427275377Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427311782Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[1]R flush_numbers[0..12]: [213, 233, 246, 246, 247, 247, 241, 247, 247, 246, 245, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427343178Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[1]R generation[0..12]: [118, 130, 136, 136, 137, 137, 134, 137, 137, 136, 135, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427375973Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427410399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[2]R flush_numbers[0..12]: [213, 233, 246, 246, 247, 247, 241, 247, 247, 246, 245, 246]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427431316Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[2]R generation[0..12]: [118, 130, 136, 136, 137, 137, 134, 137, 137, 136, 135, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427456142Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427489338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427522263Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.42754565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427568557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427603232Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"":"downstairs","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427635178Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427659794Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"0","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427677632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"1","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427710008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"client":"2","":"downstairs","session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427742713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.42776796Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"87927870-13a1-4ce0-8754-33ad6d85bd36 is now active with session: 1d6fd0e4-4aca-4dc8-a068-d2349b2f145b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427786567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427812833Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744,"session_id":"1d6fd0e4-4aca-4dc8-a068-d2349b2f145b"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427841299Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} {"msg":"Activated sub_volume 87927870-13a1-4ce0-8754-33ad6d85bd36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:43.427867726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 4096 data: 9 02/30 Read block 260 len 12288 03/30 Flush 04/30 Flush 05/30 Write block 489 len 40960 data: 11 12 11 9 9 11 13 13 14 14 06/30 Read block 317 len 28672 07/30 Read block 195 len 20480 08/30 Read block 590 len 16384 09/30 Read block 242 len 8192 10/30 Read block 25 len 8192 11/30 Write block 452 len 8192 data: 15 12 12/30 Flush 13/30 Write block 181 len 28672 data: 16 14 11 9 10 11 11 14/30 Flush 15/30 Write block 201 len 20480 data: 10 9 8 8 7 16/30 Write block 66 len 32768 data: 11 11 10 11 9 8 6 7 17/30 Write block 473 len 12288 data: 12 8 8 18/30 Read block 326 len 4096 19/30 Write block 454 len 24576 data: 10 10 10 12 13 9 20/30 Write block 223 len 12288 data: 11 11 11 21/30 Read block 445 len 8192 22/30 Read block 81 len 28672 23/30 Write block 125 len 12288 data: 18 17 13 24/30 Read block 248 len 4096 25/30 Write block 244 len 40960 data: 15 16 14 14 15 13 13 8 7 8 26/30 Write block 394 len 40960 data: 8 7 9 9 9 8 9 9 10 13 27/30 Write block 206 len 4096 data: 9 28/30 Read block 198 len 40960 29/30 Read block 5 len 28672 30/30 Write block 423 len 16384 data: 11 12 14 14 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 5 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:43.448103472Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 136 136 136 246 246 246 F F F 3 060-079 136 136 136 246 246 246 T T T 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 134 134 134 241 241 241 T T T 7 140-159 137 137 137 247 247 247 F F F 8 160-179 137 137 137 247 247 247 F F F 9 180-199 138 138 138 251 251 251 F F F 10 200-219 138 138 138 248 248 248 T T T 11 220-239 136 136 136 246 246 246 T T T 12 240-259 137 137 137 247 247 247 T T T 13 260-279 134 134 134 244 244 244 F F F 14 280-299 134 134 134 244 244 244 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 135 135 135 245 245 245 F F F 18 360-379 136 136 136 246 246 246 F F F 19 380-399 136 136 136 246 246 246 T T T 20 400-419 134 134 134 241 241 241 T T T 21 420-439 132 132 132 239 239 239 T T T 22 440-459 138 138 138 250 250 250 T T T 23 460-479 124 124 124 225 225 225 T T T 24 480-499 138 138 138 250 250 250 F F F 25 500-519 136 136 136 246 246 246 F F F 26 520-539 136 136 136 246 246 246 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 136 136 136 246 246 246 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 138, Max flush: 251 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 213 213 213 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 136 136 136 246 246 246 F F F 3 060-079 139 139 139 252 252 252 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 139 139 139 252 252 252 F F F 7 140-159 137 137 137 247 247 247 F F F 8 160-179 137 137 137 247 247 247 F F F 9 180-199 138 138 138 251 251 251 F F F 10 200-219 139 139 139 252 252 252 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 139 139 139 252 252 252 F F F 13 260-279 134 134 134 244 244 244 F F F 14 280-299 134 134 134 244 244 244 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 135 135 135 245 245 245 F F F 18 360-379 136 136 136 246 246 246 F F F 19 380-399 139 139 139 252 252 252 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 139 139 139 252 252 252 F F F 23 460-479 139 139 139 252 252 252 F F F 24 480-499 138 138 138 250 250 250 F F F 25 500-519 136 136 136 246 246 246 F F F 26 520-539 136 136 136 246 246 246 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 136 136 136 246 246 246 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 139, Max flush: 252 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.722040814Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.766330562Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.767048334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.767083729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.767098916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.767114164Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} {"msg":"Crucible ce894fca-cf84-41b1-a2ab-438cbcb10626 has session id: 2c786d14-d711-49a4-9f1b-656bca5c4517","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.767634173Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Upstairs opts: Upstairs UUID: ce894fca-cf84-41b1-a2ab-438cbcb10626, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.767680136Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Crucible stats registered with UUID: ce894fca-cf84-41b1-a2ab-438cbcb10626","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.767703493Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:57.76772433Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768132954Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768157641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} {"msg":"ce894fca-cf84-41b1-a2ab-438cbcb10626 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768222832Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768245989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"0","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768275625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"0","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768290552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.76830559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768319488Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"2","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768344305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"2","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768368762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768392309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768417155Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"ds_connection connected from Ok(127.0.0.1:43799)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768444062Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:57.768547227Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:57.76859757Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"ds_connection connected from Ok(127.0.0.1:58277)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768638525Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:10:57.768677929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.768808541Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.769267208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"0","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.769451663Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"2","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.769813863Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"0","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.769922098Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"2","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.770306195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:10:57.770491189Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.769918353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"ds_connection connected from Ok(127.0.0.1:39946)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.770128164Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.771060526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.77161019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[0]R flush_numbers[0..12]: [213, 233, 246, 252, 247, 247, 252, 247, 247, 251, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772224266Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[0]R generation[0..12]: [118, 130, 136, 139, 137, 137, 139, 137, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772249502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.77226566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[1]R flush_numbers[0..12]: [213, 233, 246, 252, 247, 247, 252, 247, 247, 251, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772286257Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[1]R generation[0..12]: [118, 130, 136, 139, 137, 137, 139, 137, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772301105Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772315543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[2]R flush_numbers[0..12]: [213, 233, 246, 252, 247, 247, 252, 247, 247, 251, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772330491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[2]R generation[0..12]: [118, 130, 136, 139, 137, 137, 139, 137, 137, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772344789Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772359277Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772372445Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772386033Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772399262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.77241266Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"":"downstairs","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772426308Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772438656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"0","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:11:07.772452564Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"1","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772469652Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"client":"2","":"downstairs","session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.7724834Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772497898Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"ce894fca-cf84-41b1-a2ab-438cbcb10626 is now active with session: 2c786d14-d711-49a4-9f1b-656bca5c4517","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772511496Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772523854Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752,"session_id":"2c786d14-d711-49a4-9f1b-656bca5c4517"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772536413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} {"msg":"Activated sub_volume ce894fca-cf84-41b1-a2ab-438cbcb10626","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:07.772547931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 13 len 12288 data: 11 11 8 02/30 Read block 500 len 8192 03/30 Read block 61 len 28672 04/30 Read block 282 len 16384 05/30 Read block 341 len 28672 06/30 Write block 245 len 32768 data: 17 15 15 16 14 14 9 8 07/30 Write block 264 len 8192 data: 15 15 08/30 Flush 09/30 Write block 62 len 28672 data: 15 17 14 15 12 12 11 10/30 Read block 554 len 24576 11/30 Read block 144 len 36864 12/30 Read block 6 len 32768 13/30 Write block 181 len 24576 data: 17 15 12 10 11 12 14/30 Read block 317 len 28672 15/30 Flush 16/30 Flush 17/30 Write block 510 len 12288 data: 13 14 16 18/30 Read block 219 len 12288 19/30 Read block 484 len 8192 20/30 Write block 120 len 12288 data: 14 16 17 21/30 Write block 265 len 8192 data: 16 15 22/30 Write block 246 len 24576 data: 16 16 17 15 15 10 23/30 Read block 68 len 16384 24/30 Read block 3 len 20480 25/30 Read block 333 len 36864 26/30 Read block 268 len 24576 27/30 Read block 579 len 32768 28/30 Read block 292 len 24576 29/30 Write block 58 len 40960 data: 11 13 12 15 16 18 15 16 13 13 30/30 Write block 150 len 28672 data: 13 12 12 11 15 16 18 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:07.793850501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 136 136 136 246 246 246 T T T 3 060-079 140 140 140 254 254 254 T T T 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 139 139 139 252 252 252 T T T 7 140-159 137 137 137 247 247 247 T T T 8 160-179 137 137 137 247 247 247 F F F 9 180-199 140 140 140 254 254 254 F F F 10 200-219 139 139 139 252 252 252 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 140 140 140 253 253 253 T T T 13 260-279 140 140 140 253 253 253 T T T 14 280-299 134 134 134 244 244 244 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 135 135 135 245 245 245 F F F 18 360-379 136 136 136 246 246 246 F F F 19 380-399 139 139 139 252 252 252 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 139 139 139 252 252 252 F F F 23 460-479 139 139 139 252 252 252 F F F 24 480-499 138 138 138 250 250 250 F F F 25 500-519 136 136 136 246 246 246 T T T 26 520-539 136 136 136 246 246 246 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 136 136 136 246 246 246 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 140, Max flush: 254 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 141 141 141 255 255 255 F F F 3 060-079 141 141 141 255 255 255 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 141 141 141 255 255 255 F F F 7 140-159 141 141 141 255 255 255 F F F 8 160-179 137 137 137 247 247 247 F F F 9 180-199 140 140 140 254 254 254 F F F 10 200-219 139 139 139 252 252 252 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 141 141 141 255 255 255 F F F 13 260-279 141 141 141 255 255 255 F F F 14 280-299 134 134 134 244 244 244 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 134 134 134 242 242 242 F F F 17 340-359 135 135 135 245 245 245 F F F 18 360-379 136 136 136 246 246 246 F F F 19 380-399 139 139 139 252 252 252 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 139 139 139 252 252 252 F F F 23 460-479 139 139 139 252 252 252 F F F 24 480-499 138 138 138 250 250 250 F F F 25 500-519 141 141 141 255 255 255 F F F 26 520-539 136 136 136 246 246 246 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 136 136 136 246 246 246 F F F 29 580-599 130 130 130 233 233 233 F F F Max gen: 141, Max flush: 255 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.053522751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.098035578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.098877302Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.098912988Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.098927166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.098942273Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} {"msg":"Crucible 1a5d5734-8e56-4862-ad21-b6be2b764b45 has session id: bd21b01a-b55d-4618-a157-b02c011124ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.099492408Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Upstairs opts: Upstairs UUID: 1a5d5734-8e56-4862-ad21-b6be2b764b45, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.099545201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Crucible stats registered with UUID: 1a5d5734-8e56-4862-ad21-b6be2b764b45","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.099575776Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:12.099602223Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100115392Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100142888Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} {"msg":"1a5d5734-8e56-4862-ad21-b6be2b764b45 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100217448Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100236246Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"0","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100264702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"0","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100283339Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"1","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100298467Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"1","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100321334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.10034456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100365908Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100390784Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100415981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:41439)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100519556Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:52974)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.100741806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:12.100921241Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:12.101024517Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:12.101061142Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.101085099Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.101215121Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"0","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.101243317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"1","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.101864042Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"1","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.101892847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"0","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.102440842Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:12.102567745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.101394282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:54505)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.101526964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.10264609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103165569Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[0]R flush_numbers[0..12]: [253, 233, 255, 255, 247, 247, 255, 255, 247, 254, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103756297Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[0]R generation[0..12]: [140, 130, 141, 141, 137, 137, 141, 141, 137, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103781164Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103797782Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[1]R flush_numbers[0..12]: [253, 233, 255, 255, 247, 247, 255, 255, 247, 254, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.10381271Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[1]R generation[0..12]: [140, 130, 141, 141, 137, 137, 141, 141, 137, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103827188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103846705Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[2]R flush_numbers[0..12]: [253, 233, 255, 255, 247, 247, 255, 255, 247, 254, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103861283Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[2]R generation[0..12]: [140, 130, 141, 141, 137, 137, 141, 141, 137, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103875281Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103889669Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103902807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103916615Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103929873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103943312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"":"downstairs","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103960959Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.103976367Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"0","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-03-18T05:11:22.104011682Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"1","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.10403231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"client":"2","":"downstairs","session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.104049607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.104065235Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"1a5d5734-8e56-4862-ad21-b6be2b764b45 is now active with session: bd21b01a-b55d-4618-a157-b02c011124ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.104081563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.10409963Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760,"session_id":"bd21b01a-b55d-4618-a157-b02c011124ad"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.104115828Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} {"msg":"Activated sub_volume 1a5d5734-8e56-4862-ad21-b6be2b764b45","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:22.104131076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 58 len 32768 data: 12 14 13 16 17 19 16 17 02/30 Write block 278 len 28672 data: 9 11 13 12 12 11 8 03/30 Flush 04/30 Write block 335 len 24576 data: 13 16 15 17 16 16 05/30 Read block 446 len 36864 06/30 Write block 571 len 16384 data: 11 10 7 7 07/30 Read block 169 len 16384 08/30 Read block 211 len 4096 09/30 Read block 47 len 4096 10/30 Read block 494 len 40960 11/30 Write block 560 len 24576 data: 10 11 11 13 14 14 12/30 Write block 367 len 12288 data: 11 11 13 13/30 Flush 14/30 Write block 566 len 40960 data: 13 12 13 11 10 12 11 8 8 9 15/30 Write block 522 len 4096 data: 12 16/30 Write block 468 len 36864 data: 9 9 10 13 12 13 9 9 7 17/30 Read block 61 len 4096 18/30 Read block 31 len 8192 19/30 Write block 540 len 32768 data: 12 11 10 9 7 9 7 9 20/30 Write block 585 len 20480 data: 9 9 9 10 12 21/30 Write block 258 len 24576 data: 17 19 19 16 17 15 22/30 Read block 115 len 4096 23/30 Read block 114 len 24576 24/30 Read block 254 len 4096 25/30 Write block 581 len 20480 data: 8 7 7 7 10 26/30 Read block 392 len 28672 27/30 Write block 335 len 20480 data: 14 17 16 18 17 28/30 Read block 159 len 36864 29/30 Write block 379 len 8192 data: 9 11 30/30 Read block 195 len 28672 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 17 16 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:22.124703566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 142 142 142 256 256 256 F F F 3 060-079 142 142 142 256 256 256 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 141 141 141 255 255 255 F F F 7 140-159 141 141 141 255 255 255 F F F 8 160-179 137 137 137 247 247 247 F F F 9 180-199 140 140 140 254 254 254 F F F 10 200-219 139 139 139 252 252 252 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 141 141 141 255 255 255 T T T 13 260-279 142 142 142 256 256 256 T T T 14 280-299 142 142 142 256 256 256 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 142 142 142 257 257 257 T T T 17 340-359 142 142 142 257 257 257 F F F 18 360-379 142 142 142 257 257 257 T T T 19 380-399 139 139 139 252 252 252 T T T 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 139 139 139 252 252 252 F F F 23 460-479 139 139 139 252 252 252 T T T 24 480-499 138 138 138 250 250 250 F F F 25 500-519 141 141 141 255 255 255 F F F 26 520-539 136 136 136 246 246 246 T T T 27 540-559 132 132 132 239 239 239 T T T 28 560-579 142 142 142 257 257 257 T T T 29 580-599 130 130 130 233 233 233 T T T Max gen: 142, Max flush: 257 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 142 142 142 256 256 256 F F F 3 060-079 142 142 142 256 256 256 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 141 141 141 255 255 255 F F F 7 140-159 141 141 141 255 255 255 F F F 8 160-179 137 137 137 247 247 247 F F F 9 180-199 140 140 140 254 254 254 F F F 10 200-219 139 139 139 252 252 252 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 143 143 143 258 258 258 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 142 142 142 256 256 256 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 143 143 143 258 258 258 F F F 17 340-359 142 142 142 257 257 257 F F F 18 360-379 143 143 143 258 258 258 F F F 19 380-399 143 143 143 258 258 258 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 139 139 139 252 252 252 F F F 23 460-479 143 143 143 258 258 258 F F F 24 480-499 138 138 138 250 250 250 F F F 25 500-519 141 141 141 255 255 255 F F F 26 520-539 143 143 143 258 258 258 F F F 27 540-559 143 143 143 258 258 258 F F F 28 560-579 143 143 143 258 258 258 F F F 29 580-599 143 143 143 258 258 258 F F F Max gen: 143, Max flush: 258 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.400697668Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.446165154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.447008528Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.447044533Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.447058561Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.447074729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} {"msg":"Crucible 62e1a2a6-bf45-4390-a9fd-ef7e97be75c8 has session id: 87f88544-5fae-4082-9d97-69ade5db89a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.447770913Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Upstairs opts: Upstairs UUID: 62e1a2a6-bf45-4390-a9fd-ef7e97be75c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.447805238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Crucible stats registered with UUID: 62e1a2a6-bf45-4390-a9fd-ef7e97be75c8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.447820226Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:36.447832915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448317188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448335985Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} {"msg":"62e1a2a6-bf45-4390-a9fd-ef7e97be75c8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448408975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448426093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"0","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448454679Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"0","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448470197Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448488704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448502633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"2","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448517551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"2","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448569133Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.44859342Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448618187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41291)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448644323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47828)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.44867001Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:36.448797032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:36.448836027Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:36.448870132Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.448984526Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.449459571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"0","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.449498056Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"2","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.450332221Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"0","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.450385594Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"2","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.451190683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:36.451273711Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.450056972Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63832)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.450190664Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.451363082Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.451941963Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[0]R flush_numbers[0..12]: [253, 233, 256, 256, 247, 247, 255, 255, 247, 254, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.45254511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[0]R generation[0..12]: [140, 130, 142, 142, 137, 137, 141, 141, 137, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452570986Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452590483Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[1]R flush_numbers[0..12]: [253, 233, 256, 256, 247, 247, 255, 255, 247, 254, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452608361Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[1]R generation[0..12]: [140, 130, 142, 142, 137, 137, 141, 141, 137, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452626289Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452644206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[2]R flush_numbers[0..12]: [253, 233, 256, 256, 247, 247, 255, 255, 247, 254, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452667413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[2]R generation[0..12]: [140, 130, 142, 142, 137, 137, 141, 141, 137, 140, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452685621Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452703708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452720146Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452738463Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452755361Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452772378Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"":"downstairs","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:11:46.452789736Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452811463Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"0","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452828671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"1","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452846338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"client":"2","":"downstairs","session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452860046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452872575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"62e1a2a6-bf45-4390-a9fd-ef7e97be75c8 is now active with session: 87f88544-5fae-4082-9d97-69ade5db89a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452885753Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452897961Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768,"session_id":"87f88544-5fae-4082-9d97-69ade5db89a6"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452912489Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} {"msg":"Activated sub_volume 62e1a2a6-bf45-4390-a9fd-ef7e97be75c8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:46.452924208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 458 len 8192 data: 14 10 02/30 Read block 46 len 32768 03/30 Read block 182 len 24576 04/30 Read block 532 len 40960 05/30 Read block 231 len 28672 06/30 Read block 308 len 8192 07/30 Flush 08/30 Write block 494 len 20480 data: 12 14 14 15 15 09/30 Flush 10/30 Write block 252 len 32768 data: 9 9 9 13 14 16 18 20 11/30 Write block 50 len 32768 data: 12 13 12 14 14 16 14 14 12/30 Read block 37 len 16384 13/30 Write block 56 len 4096 data: 15 14/30 Flush 15/30 Write block 381 len 32768 data: 11 11 9 7 7 8 8 10 16/30 Read block 185 len 32768 17/30 Write block 144 len 20480 data: 9 12 12 13 13 18/30 Read block 294 len 24576 19/30 Write block 207 len 40960 data: 10 11 13 13 10 12 15 16 14 12 20/30 Read block 266 len 8192 21/30 Read block 407 len 8192 22/30 Read block 462 len 16384 23/30 Write block 134 len 4096 data: 12 24/30 Read block 343 len 16384 25/30 Read block 195 len 36864 26/30 Write block 156 len 20480 data: 19 17 16 13 13 27/30 Write block 198 len 28672 data: 11 11 10 11 10 9 9 28/30 Write block 287 len 8192 data: 13 11 29/30 Read block 341 len 32768 30/30 Read block 552 len 12288 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 8 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 15 16 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:46.473791147Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1768} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 144 144 144 261 261 261 F F F 3 060-079 142 142 142 256 256 256 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 141 141 141 255 255 255 T T T 7 140-159 141 141 141 255 255 255 T T T 8 160-179 137 137 137 247 247 247 T T T 9 180-199 140 140 140 254 254 254 T T T 10 200-219 139 139 139 252 252 252 T T T 11 220-239 139 139 139 252 252 252 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 142 142 142 256 256 256 T T T 15 300-319 136 136 136 246 246 246 F F F 16 320-339 143 143 143 258 258 258 F F F 17 340-359 142 142 142 257 257 257 F F F 18 360-379 143 143 143 258 258 258 F F F 19 380-399 143 143 143 258 258 258 T T T 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 144 144 144 259 259 259 F F F 23 460-479 143 143 143 258 258 258 F F F 24 480-499 144 144 144 260 260 260 F F F 25 500-519 141 141 141 255 255 255 F F F 26 520-539 143 143 143 258 258 258 F F F 27 540-559 143 143 143 258 258 258 F F F 28 560-579 143 143 143 258 258 258 F F F 29 580-599 143 143 143 258 258 258 F F F Max gen: 144, Max flush: 261 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 144 144 144 261 261 261 F F F 3 060-079 142 142 142 256 256 256 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 137 137 137 247 247 247 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 145 145 145 262 262 262 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 F F F 10 200-219 145 145 145 262 262 262 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 145 145 145 262 262 262 F F F 15 300-319 136 136 136 246 246 246 F F F 16 320-339 143 143 143 258 258 258 F F F 17 340-359 142 142 142 257 257 257 F F F 18 360-379 143 143 143 258 258 258 F F F 19 380-399 145 145 145 262 262 262 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 144 144 144 259 259 259 F F F 23 460-479 143 143 143 258 258 258 F F F 24 480-499 144 144 144 260 260 260 F F F 25 500-519 141 141 141 255 255 255 F F F 26 520-539 143 143 143 258 258 258 F F F 27 540-559 143 143 143 258 258 258 F F F 28 560-579 143 143 143 258 258 258 F F F 29 580-599 143 143 143 258 258 258 F F F Max gen: 145, Max flush: 262 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.731492707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.776895631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.777653907Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.777689372Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.77770801Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.777723008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} {"msg":"Crucible e59b7f1b-5bcd-495e-83ae-687b5d72fbaa has session id: 0316aeff-b767-4428-af3a-baec7eafdc00","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.778328094Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Upstairs opts: Upstairs UUID: e59b7f1b-5bcd-495e-83ae-687b5d72fbaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.778370608Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Crucible stats registered with UUID: e59b7f1b-5bcd-495e-83ae-687b5d72fbaa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.778394025Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:50.778414702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.778948239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.778991713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} {"msg":"e59b7f1b-5bcd-495e-83ae-687b5d72fbaa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.77908706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779105447Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"0","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779134803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"0","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779150671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"1","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779165999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"1","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779194265Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"2","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779269985Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"2","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.77930434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779334396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779362702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"ds_connection connected from Ok(127.0.0.1:44451)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779389898Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"ds_connection connected from Ok(127.0.0.1:50695)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779415455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"ds_connection connected from Ok(127.0.0.1:41037)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.779442851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.780201197Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"0","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.780241051Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"1","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.780882593Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"1","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.781024383Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"0","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.781519785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.781620981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"2","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.781723257Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782182504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"2","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[0]R flush_numbers[0..12]: [253, 233, 261, 256, 247, 247, 262, 262, 262, 262, 262, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782688455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[0]R generation[0..12]: [140, 130, 144, 142, 137, 137, 145, 145, 145, 145, 145, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782713941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782730289Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[1]R flush_numbers[0..12]: [253, 233, 261, 256, 247, 247, 262, 262, 262, 262, 262, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782744857Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[1]R generation[0..12]: [140, 130, 144, 142, 137, 137, 145, 145, 145, 145, 145, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782759115Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782773483Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[2]R flush_numbers[0..12]: [253, 233, 261, 256, 247, 247, 262, 262, 262, 262, 262, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782787781Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[2]R generation[0..12]: [140, 130, 144, 142, 137, 137, 145, 145, 145, 145, 145, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782801759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782815877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782829215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782842633Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782855891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.78286915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"":"downstairs","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782886897Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782899916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"0","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782914154Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"1","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782927882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"client":"2","":"downstairs","session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:11:50.78294175Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782962217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"e59b7f1b-5bcd-495e-83ae-687b5d72fbaa is now active with session: 0316aeff-b767-4428-af3a-baec7eafdc00","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782975285Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.782987553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776,"session_id":"0316aeff-b767-4428-af3a-baec7eafdc00"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.783000592Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} {"msg":"Activated sub_volume e59b7f1b-5bcd-495e-83ae-687b5d72fbaa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:11:50.78301216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 363 len 4096 data: 6 02/30 Write block 10 len 4096 data: 11 03/30 Write block 587 len 28672 data: 10 11 13 14 12 14 14 04/30 Read block 225 len 40960 05/30 Write block 107 len 20480 data: 13 13 13 10 10 06/30 Read block 253 len 8192 07/30 Write block 67 len 20480 data: 14 12 12 10 9 08/30 Write block 329 len 20480 data: 11 13 13 11 13 09/30 Read block 162 len 12288 10/30 Read block 79 len 40960 11/30 Read block 75 len 28672 12/30 Read block 528 len 12288 13/30 Write block 316 len 32768 data: 9 7 9 8 7 9 8 8 14/30 Read block 377 len 36864 15/30 Write block 590 len 40960 data: 15 13 15 15 15 12 10 9 8 6 16/30 Read block 214 len 20480 17/30 Write block 513 len 24576 data: 17 19 19 16 14 14 18/30 Read block 481 len 12288 19/30 Write block 2 len 20480 data: 3 4 4 5 7 20/30 Write block 59 len 36864 data: 15 14 17 18 20 17 18 14 15 21/30 Write block 394 len 4096 data: 9 22/30 Flush 23/30 Write block 319 len 24576 data: 9 8 10 9 9 9 24/30 Flush 25/30 Write block 66 len 40960 data: 15 16 13 13 11 10 7 8 5 6 26/30 Read block 282 len 20480 27/30 Write block 193 len 40960 data: 15 12 11 13 9 12 12 11 12 11 28/30 Write block 76 len 4096 data: 11 29/30 Write block 346 len 4096 data: 14 30/30 Read block 319 len 12288 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:11:50.804633376Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1776} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 263 263 263 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 146 146 146 263 263 263 T T T 4 080-099 137 137 137 247 247 247 F F F 5 100-119 146 146 146 263 263 263 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 145 145 145 262 262 262 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 145 145 145 262 262 262 T T T 10 200-219 145 145 145 262 262 262 T T T 11 220-239 139 139 139 252 252 252 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 145 145 145 262 262 262 F F F 15 300-319 146 146 146 264 264 264 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 142 142 142 257 257 257 T T T 18 360-379 146 146 146 263 263 263 F F F 19 380-399 146 146 146 263 263 263 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 144 144 144 259 259 259 F F F 23 460-479 143 143 143 258 258 258 F F F 24 480-499 144 144 144 260 260 260 F F F 25 500-519 146 146 146 263 263 263 F F F 26 520-539 143 143 143 258 258 258 F F F 27 540-559 143 143 143 258 258 258 F F F 28 560-579 143 143 143 258 258 258 F F F 29 580-599 146 146 146 263 263 263 F F F Max gen: 146, Max flush: 264 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 263 263 263 F F F 1 020-039 130 130 130 233 233 233 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 147 147 147 265 265 265 F F F 4 080-099 137 137 137 247 247 247 F F F 5 100-119 146 146 146 263 263 263 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 145 145 145 262 262 262 F F F 8 160-179 145 145 145 262 262 262 F F F 9 180-199 147 147 147 265 265 265 F F F 10 200-219 147 147 147 265 265 265 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 145 145 145 262 262 262 F F F 15 300-319 146 146 146 264 264 264 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 147 147 147 265 265 265 F F F 18 360-379 146 146 146 263 263 263 F F F 19 380-399 146 146 146 263 263 263 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 144 144 144 259 259 259 F F F 23 460-479 143 143 143 258 258 258 F F F 24 480-499 144 144 144 260 260 260 F F F 25 500-519 146 146 146 263 263 263 F F F 26 520-539 143 143 143 258 258 258 F F F 27 540-559 143 143 143 258 258 258 F F F 28 560-579 143 143 143 258 258 258 F F F 29 580-599 146 146 146 263 263 263 F F F Max gen: 147, Max flush: 265 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.047501382Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:05.092205034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.092652092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.124081529Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.124787572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.124830796Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.124847704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.124865981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} {"msg":"Crucible 74a8c582-f095-4e54-b319-302f34d64243 has session id: d49fcdb2-fc27-49f1-b90b-210d934410f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.12545552Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Upstairs opts: Upstairs UUID: 74a8c582-f095-4e54-b319-302f34d64243, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.125496415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Crucible stats registered with UUID: 74a8c582-f095-4e54-b319-302f34d64243","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.125512622Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:05.12553016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.125927875Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.125952592Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} {"msg":"74a8c582-f095-4e54-b319-302f34d64243 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126027511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126054868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"0","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126095772Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"0","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126124228Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"1","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126153964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"1","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.12618127Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"2","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126212896Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"2","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126241842Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126272208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126300084Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36901)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.12632875Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:59589)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126356056Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60770)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.126384952Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.127067748Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"1","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.127110453Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"2","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.127143398Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"0","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.127736307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"2","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.127889746Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"1","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.12793089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"0","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128285431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128535597Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[0]R flush_numbers[0..12]: [263, 233, 263, 265, 247, 263, 262, 262, 262, 265, 265, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128678487Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[0]R generation[0..12]: [146, 130, 146, 147, 137, 146, 145, 145, 145, 147, 147, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128706423Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128723061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[1]R flush_numbers[0..12]: [263, 233, 263, 265, 247, 263, 262, 262, 262, 265, 265, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128737579Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[1]R generation[0..12]: [146, 130, 146, 147, 137, 146, 145, 145, 145, 147, 147, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128751957Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128772224Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[2]R flush_numbers[0..12]: [263, 233, 263, 265, 247, 263, 262, 262, 262, 265, 265, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128787272Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[2]R generation[0..12]: [146, 130, 146, 147, 137, 146, 145, 145, 145, 147, 147, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.12880151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128816648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128838125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Generation requested: 148 >= found:148",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.12887092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128895787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128920634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"":"downstairs","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.12894554Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128968677Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"0","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.128989204Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"1","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.129024779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"client":"2","":"downstairs","session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.129039667Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.129052426Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"74a8c582-f095-4e54-b319-302f34d64243 is now active with session: d49fcdb2-fc27-49f1-b90b-210d934410f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.129065764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.129081171Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"d49fcdb2-fc27-49f1-b90b-210d934410f7"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.129097479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} {"msg":"Activated sub_volume 74a8c582-f095-4e54-b319-302f34d64243","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:05.129109228Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 288 len 20480 data: 12 10 10 7 6 02/30 Write block 95 len 12288 data: 13 14 11 03/30 Read block 495 len 16384 04/30 Read block 250 len 16384 05/30 Write block 542 len 20480 data: 11 10 8 10 8 06/30 Read block 167 len 36864 07/30 Read block 304 len 32768 08/30 Read block 572 len 20480 09/30 Write block 212 len 8192 data: 13 16 10/30 Write block 549 len 16384 data: 11 12 12 12 11/30 Read block 43 len 36864 12/30 Read block 93 len 36864 13/30 Read block 425 len 40960 14/30 Read block 288 len 32768 15/30 Write block 593 len 24576 data: 16 16 13 11 10 9 16/30 Write block 231 len 28672 data: 12 14 12 10 11 15 17 17/30 Read block 467 len 4096 18/30 Write block 308 len 16384 data: 7 6 7 10 19/30 Write block 497 len 36864 data: 16 16 14 14 12 13 16 13 11 20/30 Flush 21/30 Write block 62 len 4096 data: 19 22/30 Write block 20 len 28672 data: 10 8 7 8 8 7 7 23/30 Read block 260 len 8192 24/30 Read block 131 len 4096 25/30 Read block 396 len 36864 26/30 Read block 510 len 16384 27/30 Read block 329 len 32768 28/30 Write block 154 len 36864 data: 16 17 20 18 17 14 14 13 12 29/30 Read block 85 len 4096 30/30 Write block 464 len 16384 data: 6 8 9 7 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:05.150875624Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 263 263 263 F F F 1 020-039 130 130 130 233 233 233 T T T 2 040-059 146 146 146 263 263 263 F F F 3 060-079 147 147 147 265 265 265 T T T 4 080-099 148 148 148 266 266 266 F F F 5 100-119 146 146 146 263 263 263 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 145 145 145 262 262 262 T T T 8 160-179 145 145 145 262 262 262 T T T 9 180-199 147 147 147 265 265 265 F F F 10 200-219 148 148 148 266 266 266 F F F 11 220-239 148 148 148 266 266 266 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 148 148 148 266 266 266 F F F 15 300-319 148 148 148 266 266 266 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 147 147 147 265 265 265 F F F 18 360-379 146 146 146 263 263 263 F F F 19 380-399 146 146 146 263 263 263 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 144 144 144 259 259 259 F F F 23 460-479 143 143 143 258 258 258 T T T 24 480-499 148 148 148 266 266 266 F F F 25 500-519 148 148 148 266 266 266 F F F 26 520-539 143 143 143 258 258 258 F F F 27 540-559 148 148 148 266 266 266 F F F 28 560-579 143 143 143 258 258 258 F F F 29 580-599 148 148 148 266 266 266 F F F Max gen: 148, Max flush: 266 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 263 263 263 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 149 149 149 267 267 267 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 146 146 146 263 263 263 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 147 147 147 265 265 265 F F F 10 200-219 148 148 148 266 266 266 F F F 11 220-239 148 148 148 266 266 266 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 148 148 148 266 266 266 F F F 15 300-319 148 148 148 266 266 266 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 147 147 147 265 265 265 F F F 18 360-379 146 146 146 263 263 263 F F F 19 380-399 146 146 146 263 263 263 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 144 144 144 259 259 259 F F F 23 460-479 149 149 149 267 267 267 F F F 24 480-499 148 148 148 266 266 266 F F F 25 500-519 148 148 148 266 266 266 F F F 26 520-539 143 143 143 258 258 258 F F F 27 540-559 148 148 148 266 266 266 F F F 28 560-579 143 143 143 258 258 258 F F F 29 580-599 148 148 148 266 266 266 F F F Max gen: 149, Max flush: 267 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.428398947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.473850596Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.474588384Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.474633328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.474665853Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.47468713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} {"msg":"Crucible fe909716-f6d7-4869-b0df-5e835ad29ef6 has session id: 377d5ee1-009c-4882-8f43-7cf6a9aed16b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.475186512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Upstairs opts: Upstairs UUID: fe909716-f6d7-4869-b0df-5e835ad29ef6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.475227166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Crucible stats registered with UUID: fe909716-f6d7-4869-b0df-5e835ad29ef6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.475243514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:09.475265531Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.475807407Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.475831613Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} {"msg":"fe909716-f6d7-4869-b0df-5e835ad29ef6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.475906143Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.47592641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"0","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.475972504Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"0","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.47599967Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"1","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476029726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"1","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476057352Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"2","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476091357Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"2","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476122313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476151829Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476179945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44760)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476208191Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45132)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476232208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44291)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476448678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476736249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"2","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.476765505Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"0","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.477318848Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"0","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.477441592Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"2","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.477990476Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.478130497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.47832325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"1","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.47883436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"1","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[0]R flush_numbers[0..12]: [263, 267, 263, 267, 266, 263, 262, 267, 267, 265, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479336291Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[0]R generation[0..12]: [146, 149, 146, 149, 148, 146, 145, 149, 149, 147, 148, 148]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479356128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479371396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[1]R flush_numbers[0..12]: [263, 267, 263, 267, 266, 263, 262, 267, 267, 265, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479386024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[1]R generation[0..12]: [146, 149, 146, 149, 148, 146, 145, 149, 149, 147, 148, 148]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479400382Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.47941493Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[2]R flush_numbers[0..12]: [263, 267, 263, 267, 266, 263, 262, 267, 267, 265, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479429298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[2]R generation[0..12]: [146, 149, 146, 149, 148, 146, 145, 149, 149, 147, 148, 148]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479443446Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479458374Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479472322Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479486141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479499729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479513367Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"":"downstairs","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479530144Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479543063Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"0","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:12:09.479556991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"1","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479577008Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"client":"2","":"downstairs","session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479591076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479603664Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"fe909716-f6d7-4869-b0df-5e835ad29ef6 is now active with session: 377d5ee1-009c-4882-8f43-7cf6a9aed16b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479617012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479629511Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792,"session_id":"377d5ee1-009c-4882-8f43-7cf6a9aed16b"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479642199Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} {"msg":"Activated sub_volume fe909716-f6d7-4869-b0df-5e835ad29ef6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:09.479653827Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Flush 04/30 Write block 306 len 8192 data: 5 5 05/30 Read block 345 len 16384 06/30 Read block 118 len 32768 07/30 Write block 584 len 16384 data: 8 11 10 11 08/30 Write block 499 len 24576 data: 15 15 13 14 17 14 09/30 Flush 10/30 Write block 530 len 40960 data: 18 17 16 16 15 13 13 14 12 15 11/30 Read block 367 len 20480 12/30 Write block 219 len 32768 data: 9 11 11 12 12 12 12 13 13/30 Read block 103 len 24576 14/30 Read block 334 len 40960 15/30 Flush 16/30 Write block 440 len 32768 data: 8 11 10 12 14 15 16 19 17/30 Write block 68 len 24576 data: 14 14 12 11 8 9 18/30 Read block 36 len 36864 19/30 Read block 397 len 4096 20/30 Read block 197 len 4096 21/30 Read block 76 len 28672 22/30 Read block 239 len 32768 23/30 Read block 548 len 32768 24/30 Read block 553 len 24576 25/30 Read block 177 len 16384 26/30 Write block 573 len 28672 data: 9 9 10 7 8 11 12 27/30 Read block 42 len 4096 28/30 Write block 2 len 12288 data: 4 5 5 29/30 Read block 280 len 24576 30/30 Write block 466 len 40960 data: 10 8 10 10 11 14 13 14 10 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:09.501856424Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1792} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 263 263 263 T T T 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 149 149 149 267 267 267 T T T 4 080-099 148 148 148 266 266 266 F F F 5 100-119 146 146 146 263 263 263 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 147 147 147 265 265 265 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 150 150 150 272 272 272 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 148 148 148 266 266 266 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 147 147 147 265 265 265 F F F 18 360-379 146 146 146 263 263 263 F F F 19 380-399 146 146 146 263 263 263 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 144 144 144 259 259 259 T T T 23 460-479 149 149 149 267 267 267 T T T 24 480-499 150 150 150 271 271 271 F F F 25 500-519 150 150 150 271 271 271 F F F 26 520-539 150 150 150 272 272 272 F F F 27 540-559 148 148 148 266 266 266 F F F 28 560-579 143 143 143 258 258 258 T T T 29 580-599 150 150 150 271 271 271 F F F Max gen: 150, Max flush: 272 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 273 273 273 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 151 151 151 273 273 273 F F F 4 080-099 148 148 148 266 266 266 F F F 5 100-119 146 146 146 263 263 263 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 147 147 147 265 265 265 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 150 150 150 272 272 272 F F F 12 240-259 144 144 144 261 261 261 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 148 148 148 266 266 266 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 147 147 147 265 265 265 F F F 18 360-379 146 146 146 263 263 263 F F F 19 380-399 146 146 146 263 263 263 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 151 151 151 273 273 273 F F F 23 460-479 151 151 151 273 273 273 F F F 24 480-499 150 150 150 271 271 271 F F F 25 500-519 150 150 150 271 271 271 F F F 26 520-539 150 150 150 272 272 272 F F F 27 540-559 148 148 148 266 266 266 F F F 28 560-579 151 151 151 273 273 273 F F F 29 580-599 150 150 150 271 271 271 F F F Max gen: 151, Max flush: 273 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.752676086Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.797234807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.798033437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.798076061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.798089949Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.798104537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} {"msg":"Crucible 9eb61f44-f110-4970-9101-479fbc8e6c50 has session id: 44a12a84-23bb-4990-8de2-25d3cb571576","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.798788453Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Upstairs opts: Upstairs UUID: 9eb61f44-f110-4970-9101-479fbc8e6c50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.798835317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Crucible stats registered with UUID: 9eb61f44-f110-4970-9101-479fbc8e6c50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.798860663Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:23.79888166Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799344167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799377422Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} {"msg":"9eb61f44-f110-4970-9101-479fbc8e6c50 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799433724Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799460801Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"0","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799492226Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"0","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799507874Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799523012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.79953696Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"2","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799551598Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"2","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799565696Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799587383Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.79961176Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"ds_connection connected from Ok(127.0.0.1:35410)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799636116Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"ds_connection connected from Ok(127.0.0.1:33487)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799660883Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:23.799690349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:23.799724104Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:23.79975402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.799924867Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.80041305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"0","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.800453624Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"2","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.801093996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"2","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.801238056Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"0","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.801660378Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:23.801782251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.801241101Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"ds_connection connected from Ok(127.0.0.1:63585)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.801338837Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.802417559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803030155Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[0]R flush_numbers[0..12]: [273, 267, 263, 273, 266, 263, 262, 267, 267, 265, 272, 272]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803604076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[0]R generation[0..12]: [151, 149, 146, 151, 148, 146, 145, 149, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803630222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.80364634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[1]R flush_numbers[0..12]: [273, 267, 263, 273, 266, 263, 262, 267, 267, 265, 272, 272]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803661508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[1]R generation[0..12]: [151, 149, 146, 151, 148, 146, 145, 149, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803675706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803690074Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[2]R flush_numbers[0..12]: [273, 267, 263, 273, 266, 263, 262, 267, 267, 265, 272, 272]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803704182Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[2]R generation[0..12]: [151, 149, 146, 151, 148, 146, 145, 149, 149, 147, 150, 150]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.80371888Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803733498Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803746936Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803760754Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803774072Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.80378836Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"":"downstairs","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803801819Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803814187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"0","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803827865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"1","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803843603Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"client":"2","":"downstairs","session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:12:33.803857571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803875088Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"9eb61f44-f110-4970-9101-479fbc8e6c50 is now active with session: 44a12a84-23bb-4990-8de2-25d3cb571576","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803888107Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803900625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800,"session_id":"44a12a84-23bb-4990-8de2-25d3cb571576"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803913183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} {"msg":"Activated sub_volume 9eb61f44-f110-4970-9101-479fbc8e6c50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:33.803924642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 537 len 28672 data: 15 13 16 13 12 12 11 02/30 Read block 187 len 36864 03/30 Write block 495 len 28672 data: 15 15 17 17 16 16 14 04/30 Read block 150 len 20480 05/30 Write block 497 len 16384 data: 18 18 17 17 06/30 Write block 349 len 24576 data: 12 12 13 14 14 14 07/30 Read block 201 len 24576 08/30 Read block 331 len 40960 09/30 Write block 562 len 36864 data: 12 14 15 15 14 13 14 12 11 10/30 Read block 324 len 36864 11/30 Write block 451 len 16384 data: 19 16 13 11 12/30 Read block 315 len 12288 13/30 Write block 462 len 20480 data: 5 5 7 9 11 14/30 Read block 73 len 8192 15/30 Write block 241 len 32768 data: 18 15 16 16 18 17 17 18 16/30 Read block 81 len 20480 17/30 Read block 45 len 4096 18/30 Read block 71 len 16384 19/30 Flush 20/30 Write block 236 len 20480 data: 16 18 15 16 17 21/30 Read block 128 len 32768 22/30 Read block 352 len 4096 23/30 Write block 379 len 28672 data: 10 12 12 12 10 8 8 24/30 Write block 96 len 28672 data: 15 12 8 11 11 10 9 25/30 Read block 541 len 8192 26/30 Write block 561 len 24576 data: 12 13 15 16 16 15 27/30 Write block 350 len 8192 data: 13 14 28/30 Read block 415 len 4096 29/30 Write block 94 len 16384 data: 15 14 16 13 30/30 Write block 72 len 24576 data: 9 10 6 7 12 12 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:33.825873563Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1800} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 273 273 273 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 151 151 151 273 273 273 T T T 4 080-099 148 148 148 266 266 266 T T T 5 100-119 146 146 146 263 263 263 T T T 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 147 147 147 265 265 265 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 150 150 150 272 272 272 T T T 12 240-259 152 152 152 274 274 274 T T T 13 260-279 143 143 143 258 258 258 F F F 14 280-299 148 148 148 266 266 266 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 152 152 152 274 274 274 T T T 18 360-379 146 146 146 263 263 263 T T T 19 380-399 146 146 146 263 263 263 T T T 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 152 152 152 274 274 274 F F F 23 460-479 152 152 152 274 274 274 F F F 24 480-499 152 152 152 274 274 274 F F F 25 500-519 152 152 152 274 274 274 F F F 26 520-539 152 152 152 274 274 274 F F F 27 540-559 152 152 152 274 274 274 F F F 28 560-579 152 152 152 274 274 274 T T T 29 580-599 150 150 150 271 271 271 F F F Max gen: 152, Max flush: 274 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 273 273 273 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 153 153 153 275 275 275 F F F 4 080-099 153 153 153 275 275 275 F F F 5 100-119 153 153 153 275 275 275 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 149 149 149 267 267 267 F F F 9 180-199 147 147 147 265 265 265 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 153 153 153 275 275 275 F F F 13 260-279 143 143 143 258 258 258 F F F 14 280-299 148 148 148 266 266 266 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 153 153 153 275 275 275 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 153 153 153 275 275 275 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 152 152 152 274 274 274 F F F 23 460-479 152 152 152 274 274 274 F F F 24 480-499 152 152 152 274 274 274 F F F 25 500-519 152 152 152 274 274 274 F F F 26 520-539 152 152 152 274 274 274 F F F 27 540-559 152 152 152 274 274 274 F F F 28 560-579 153 153 153 275 275 275 F F F 29 580-599 150 150 150 271 271 271 F F F Max gen: 153, Max flush: 275 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.091606315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:48.135620391Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.136129281Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.168356738Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.16906847Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.169103745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.169119173Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.169133981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} {"msg":"Crucible da233516-c190-48ab-938b-9aba7f48b703 has session id: a93312b8-a209-4620-a793-626082406959","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.169742648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Upstairs opts: Upstairs UUID: da233516-c190-48ab-938b-9aba7f48b703, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.169785522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Crucible stats registered with UUID: da233516-c190-48ab-938b-9aba7f48b703","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.169813248Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:48.169835045Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170325017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170356803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} {"msg":"da233516-c190-48ab-938b-9aba7f48b703 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170414355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170442381Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"0","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170479206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"0","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170495634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"1","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170510662Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"1","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.17052444Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"2","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170546597Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"2","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170574653Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170603359Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170630475Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"ds_connection connected from Ok(127.0.0.1:54757)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170753558Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"ds_connection connected from Ok(127.0.0.1:43752)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170801952Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"ds_connection connected from Ok(127.0.0.1:41026)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.170831188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.171598752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"1","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.171647675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"2","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.171680001Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"0","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.172239694Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"1","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.172348619Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"0","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.172387544Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"2","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.172915581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173047903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[0]R flush_numbers[0..12]: [273, 267, 263, 275, 275, 275, 262, 267, 267, 265, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173074739Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[0]R generation[0..12]: [151, 149, 146, 153, 153, 153, 145, 149, 149, 147, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173091467Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173106435Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[1]R flush_numbers[0..12]: [273, 267, 263, 275, 275, 275, 262, 267, 267, 265, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173121153Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[1]R generation[0..12]: [151, 149, 146, 153, 153, 153, 145, 149, 149, 147, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173135371Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173149879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[2]R flush_numbers[0..12]: [273, 267, 263, 275, 275, 275, 262, 267, 267, 265, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173169456Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[2]R generation[0..12]: [151, 149, 146, 153, 153, 153, 145, 149, 149, 147, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173184664Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173199442Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.17321277Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173226449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173239637Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173253025Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"":"downstairs","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173266743Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173278931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"0","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173292889Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"1","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173306678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"client":"2","":"downstairs","session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173320516Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173333174Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"da233516-c190-48ab-938b-9aba7f48b703 is now active with session: a93312b8-a209-4620-a793-626082406959","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:12:48.173346322Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.17336387Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808,"session_id":"a93312b8-a209-4620-a793-626082406959"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173376378Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} {"msg":"Activated sub_volume da233516-c190-48ab-938b-9aba7f48b703","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:48.173389886Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 481 len 4096 data: 10 02/30 Flush 03/30 Write block 439 len 36864 data: 7 9 12 11 13 15 16 17 20 04/30 Write block 447 len 36864 data: 21 21 21 19 20 17 14 12 11 05/30 Write block 596 len 12288 data: 12 11 10 06/30 Read block 112 len 24576 07/30 Write block 179 len 24576 data: 12 15 18 16 13 11 08/30 Write block 113 len 20480 data: 8 7 10 11 12 09/30 Read block 181 len 24576 10/30 Write block 578 len 8192 data: 12 13 11/30 Write block 106 len 16384 data: 11 14 14 14 12/30 Write block 278 len 28672 data: 10 12 14 13 13 12 9 13/30 Flush 14/30 Write block 9 len 28672 data: 8 12 10 11 12 12 9 15/30 Read block 225 len 20480 16/30 Write block 435 len 4096 data: 10 17/30 Write block 385 len 8192 data: 9 9 18/30 Read block 465 len 16384 19/30 Read block 97 len 4096 20/30 Read block 594 len 24576 21/30 Write block 292 len 16384 data: 7 5 5 7 22/30 Write block 281 len 28672 data: 14 14 13 10 9 11 14 23/30 Write block 557 len 24576 data: 9 9 10 11 13 14 24/30 Read block 68 len 8192 25/30 Flush 26/30 Flush 27/30 Write block 351 len 28672 data: 15 15 15 15 11 13 11 28/30 Write block 94 len 8192 data: 16 15 29/30 Write block 73 len 4096 data: 11 30/30 Read block 198 len 36864 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 3 2 4 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:48.19132284Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1808} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 278 278 278 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 153 153 153 275 275 275 T T T 4 080-099 153 153 153 275 275 275 T T T 5 100-119 154 154 154 277 277 277 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 154 154 154 277 277 277 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 153 153 153 275 275 275 F F F 13 260-279 154 154 154 277 277 277 F F F 14 280-299 154 154 154 278 278 278 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 153 153 153 275 275 275 T T T 18 360-379 153 153 153 275 275 275 F F F 19 380-399 154 154 154 278 278 278 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 154 154 154 278 278 278 F F F 22 440-459 154 154 154 277 277 277 F F F 23 460-479 152 152 152 274 274 274 F F F 24 480-499 154 154 154 276 276 276 F F F 25 500-519 152 152 152 274 274 274 F F F 26 520-539 152 152 152 274 274 274 F F F 27 540-559 154 154 154 278 278 278 F F F 28 560-579 154 154 154 278 278 278 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 154, Max flush: 278 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 278 278 278 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 155 155 155 279 279 279 F F F 4 080-099 155 155 155 279 279 279 F F F 5 100-119 154 154 154 277 277 277 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 154 154 154 277 277 277 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 153 153 153 275 275 275 F F F 13 260-279 154 154 154 277 277 277 F F F 14 280-299 154 154 154 278 278 278 F F F 15 300-319 150 150 150 271 271 271 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 154 154 154 278 278 278 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 154 154 154 278 278 278 F F F 22 440-459 154 154 154 277 277 277 F F F 23 460-479 152 152 152 274 274 274 F F F 24 480-499 154 154 154 276 276 276 F F F 25 500-519 152 152 152 274 274 274 F F F 26 520-539 152 152 152 274 274 274 F F F 27 540-559 154 154 154 278 278 278 F F F 28 560-579 154 154 154 278 278 278 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 155, Max flush: 279 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.455671346Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.50064313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.501404915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.50144479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.501458658Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.501473566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} {"msg":"Crucible 597b893d-295e-4c77-b3ca-a37976295cf1 has session id: 36c1113c-6a99-4fcd-9cab-a55a5349c233","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502071664Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Upstairs opts: Upstairs UUID: 597b893d-295e-4c77-b3ca-a37976295cf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502107409Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Crucible stats registered with UUID: 597b893d-295e-4c77-b3ca-a37976295cf1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502123987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:52.502137355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502598991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502632687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} {"msg":"597b893d-295e-4c77-b3ca-a37976295cf1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502687589Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502713575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"0","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502743301Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"0","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502765548Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"1","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502792944Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"1","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502822351Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"2","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502852786Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"2","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502881632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502914518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502944754Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"ds_connection connected from Ok(127.0.0.1:54554)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.5029724Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"ds_connection connected from Ok(127.0.0.1:54203)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.502999706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"ds_connection connected from Ok(127.0.0.1:37508)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.503122689Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.50376709Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"0","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.503801756Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"1","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.504409122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"1","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.504521297Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"0","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.504638391Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"2","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.504972385Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505114625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505363521Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"2","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[0]R flush_numbers[0..12]: [278, 267, 263, 279, 279, 277, 262, 267, 277, 277, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505889739Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[0]R generation[0..12]: [154, 149, 146, 155, 155, 154, 145, 149, 154, 154, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505911506Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505930873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[1]R flush_numbers[0..12]: [278, 267, 263, 279, 279, 277, 262, 267, 277, 277, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505945701Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[1]R generation[0..12]: [154, 149, 146, 155, 155, 154, 145, 149, 154, 154, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505959579Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505973707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[2]R flush_numbers[0..12]: [278, 267, 263, 279, 279, 277, 262, 267, 277, 277, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.505987735Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[2]R generation[0..12]: [154, 149, 146, 155, 155, 154, 145, 149, 154, 154, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506017751Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506033649Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506047167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506060745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506073833Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506087111Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"":"downstairs","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.50610048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506113048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"0","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506127316Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"1","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506144164Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"client":"2","":"downstairs","session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506158232Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506179179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"597b893d-295e-4c77-b3ca-a37976295cf1 is now active with session: 36c1113c-6a99-4fcd-9cab-a55a5349c233","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506192227Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506204415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816,"session_id":"36c1113c-6a99-4fcd-9cab-a55a5349c233"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506217034Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} {"msg":"Activated sub_volume 597b893d-295e-4c77-b3ca-a37976295cf1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:52.506228522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 527 len 16384 data: 12 14 16 19 02/30 Flush 03/30 Write block 78 len 36864 data: 12 11 11 13 10 11 13 11 15 04/30 Flush 05/30 Flush 06/30 Write block 444 len 40960 data: 16 17 18 22 22 22 20 21 18 15 07/30 Write block 404 len 32768 data: 12 10 9 12 11 12 12 10 08/30 Read block 513 len 12288 09/30 Read block 126 len 16384 10/30 Flush 11/30 Flush 12/30 Write block 412 len 20480 data: 12 14 14 14 11 13/30 Flush 14/30 Write block 270 len 8192 data: 9 9 15/30 Write block 486 len 24576 data: 15 14 12 12 13 12 16/30 Write block 250 len 8192 data: 16 11 17/30 Read block 450 len 16384 18/30 Write block 269 len 32768 data: 9 10 10 7 8 6 8 8 19/30 Read block 287 len 36864 20/30 Read block 323 len 32768 21/30 Read block 519 len 8192 22/30 Write block 461 len 4096 data: 7 23/30 Read block 115 len 40960 24/30 Read block 562 len 12288 25/30 Read block 294 len 40960 26/30 Read block 272 len 12288 27/30 Write block 299 len 40960 data: 8 6 6 6 7 7 7 6 6 8 28/30 Read block 491 len 8192 29/30 Write block 267 len 20480 data: 13 13 10 11 11 30/30 Write block 104 len 4096 data: 15 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 2 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:52.526465229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1816} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 278 278 278 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 156 156 156 281 281 281 F F F 4 080-099 156 156 156 281 281 281 F F F 5 100-119 154 154 154 277 277 277 T T T 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 154 154 154 277 277 277 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 153 153 153 275 275 275 T T T 13 260-279 154 154 154 277 277 277 T T T 14 280-299 154 154 154 278 278 278 T T T 15 300-319 150 150 150 271 271 271 T T T 16 320-339 146 146 146 264 264 264 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 154 154 154 278 278 278 F F F 20 400-419 156 156 156 285 285 285 F F F 21 420-439 154 154 154 278 278 278 F F F 22 440-459 156 156 156 283 283 283 F F F 23 460-479 152 152 152 274 274 274 T T T 24 480-499 154 154 154 276 276 276 T T T 25 500-519 152 152 152 274 274 274 F F F 26 520-539 156 156 156 280 280 280 F F F 27 540-559 154 154 154 278 278 278 F F F 28 560-579 154 154 154 278 278 278 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 156, Max flush: 285 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 278 278 278 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 146 146 146 263 263 263 F F F 3 060-079 156 156 156 281 281 281 F F F 4 080-099 156 156 156 281 281 281 F F F 5 100-119 157 157 157 286 286 286 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 154 154 154 277 277 277 F F F 10 200-219 150 150 150 272 272 272 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 157 157 157 286 286 286 F F F 13 260-279 157 157 157 286 286 286 F F F 14 280-299 157 157 157 286 286 286 F F F 15 300-319 157 157 157 286 286 286 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 154 154 154 278 278 278 F F F 20 400-419 156 156 156 285 285 285 F F F 21 420-439 154 154 154 278 278 278 F F F 22 440-459 156 156 156 283 283 283 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 157 157 157 286 286 286 F F F 25 500-519 152 152 152 274 274 274 F F F 26 520-539 156 156 156 280 280 280 F F F 27 540-559 154 154 154 278 278 278 F F F 28 560-579 154 154 154 278 278 278 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 157, Max flush: 286 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.781819461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:56.826331779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.826718966Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.857779933Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.858375791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.858409947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.858423735Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.858438233Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} {"msg":"Crucible 3e16e048-8f0d-4675-a0f3-57e8e6155bcd has session id: b899642d-d485-425a-a56c-a3a8432e2c5c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859177241Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Upstairs opts: Upstairs UUID: 3e16e048-8f0d-4675-a0f3-57e8e6155bcd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859231284Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Crucible stats registered with UUID: 3e16e048-8f0d-4675-a0f3-57e8e6155bcd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.85925828Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:56.859275478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859663934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859704779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} {"msg":"3e16e048-8f0d-4675-a0f3-57e8e6155bcd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.85976577Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859793296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"0","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859837821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"0","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859880424Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"1","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.85991183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"1","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859940046Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"2","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859970222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"2","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.859998808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.860025025Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.860051061Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55607)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.860080327Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47661)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.860108173Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64387)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.860260052Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.860775981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"2","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.860807687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"1","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.86100518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"0","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.861463007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"2","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.861504291Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"1","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.861536587Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"0","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862041407Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862126656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[0]R flush_numbers[0..12]: [278, 267, 263, 281, 281, 286, 262, 267, 277, 277, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862175619Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[0]R generation[0..12]: [154, 149, 146, 156, 156, 157, 145, 149, 154, 154, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862195666Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862211514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[1]R flush_numbers[0..12]: [278, 267, 263, 281, 281, 286, 262, 267, 277, 277, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862226212Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[1]R generation[0..12]: [154, 149, 146, 156, 156, 157, 145, 149, 154, 154, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.86224076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862255788Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[2]R flush_numbers[0..12]: [278, 267, 263, 281, 281, 286, 262, 267, 277, 277, 272, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862273375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[2]R generation[0..12]: [154, 149, 146, 156, 156, 157, 145, 149, 154, 154, 150, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862292473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862318239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862342606Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862361893Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862375641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862389019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"":"downstairs","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862402898Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862415286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"0","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:12:56.862429294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"1","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862449761Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"client":"2","":"downstairs","session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862463679Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862476158Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"3e16e048-8f0d-4675-a0f3-57e8e6155bcd is now active with session: b899642d-d485-425a-a56c-a3a8432e2c5c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862489306Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862501474Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824,"session_id":"b899642d-d485-425a-a56c-a3a8432e2c5c"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.862514032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} {"msg":"Activated sub_volume 3e16e048-8f0d-4675-a0f3-57e8e6155bcd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:12:56.86252912Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 8192 data: 8 6 02/30 Write block 277 len 4096 data: 9 03/30 Read block 310 len 8192 04/30 Write block 55 len 4096 data: 17 05/30 Write block 423 len 4096 data: 12 06/30 Write block 382 len 28672 data: 13 11 9 10 10 9 11 07/30 Write block 434 len 4096 data: 9 08/30 Read block 89 len 28672 09/30 Read block 199 len 12288 10/30 Write block 42 len 24576 data: 19 18 20 18 18 17 11/30 Read block 60 len 16384 12/30 Read block 218 len 4096 13/30 Read block 279 len 36864 14/30 Flush 15/30 Write block 410 len 28672 data: 13 11 13 15 15 15 12 16/30 Write block 529 len 32768 data: 17 20 18 17 17 16 14 14 17/30 Write block 203 len 4096 data: 10 18/30 Write block 60 len 12288 data: 15 18 20 19/30 Write block 532 len 28672 data: 18 18 17 15 15 16 14 20/30 Read block 66 len 8192 21/30 Read block 451 len 8192 22/30 Read block 206 len 16384 23/30 Write block 432 len 28672 data: 7 7 10 11 9 8 7 24/30 Read block 454 len 40960 25/30 Write block 556 len 28672 data: 10 10 10 11 12 14 15 26/30 Read block 542 len 12288 27/30 Write block 489 len 20480 data: 13 14 13 10 10 28/30 Read block 194 len 32768 29/30 Read block 582 len 8192 30/30 Read block 465 len 12288 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 15 15 16 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:12:56.88150519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1824} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 278 278 278 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 158 158 158 287 287 287 F F F 3 060-079 156 156 156 281 281 281 T T T 4 080-099 156 156 156 281 281 281 F F F 5 100-119 157 157 157 286 286 286 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 154 154 154 277 277 277 F F F 10 200-219 150 150 150 272 272 272 T T T 11 220-239 153 153 153 275 275 275 F F F 12 240-259 157 157 157 286 286 286 F F F 13 260-279 158 158 158 287 287 287 F F F 14 280-299 157 157 157 286 286 286 F F F 15 300-319 157 157 157 286 286 286 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 156 156 156 285 285 285 T T T 21 420-439 158 158 158 287 287 287 T T T 22 440-459 156 156 156 283 283 283 F F F 23 460-479 158 158 158 287 287 287 F F F 24 480-499 157 157 157 286 286 286 T T T 25 500-519 152 152 152 274 274 274 F F F 26 520-539 156 156 156 280 280 280 T T T 27 540-559 154 154 154 278 278 278 T T T 28 560-579 154 154 154 278 278 278 T T T 29 580-599 154 154 154 277 277 277 F F F Max gen: 158, Max flush: 287 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 278 278 278 F F F 1 020-039 149 149 149 267 267 267 F F F 2 040-059 158 158 158 287 287 287 F F F 3 060-079 159 159 159 288 288 288 F F F 4 080-099 156 156 156 281 281 281 F F F 5 100-119 157 157 157 286 286 286 F F F 6 120-139 145 145 145 262 262 262 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 154 154 154 277 277 277 F F F 10 200-219 159 159 159 288 288 288 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 157 157 157 286 286 286 F F F 13 260-279 158 158 158 287 287 287 F F F 14 280-299 157 157 157 286 286 286 F F F 15 300-319 157 157 157 286 286 286 F F F 16 320-339 146 146 146 264 264 264 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 159 159 159 288 288 288 F F F 21 420-439 159 159 159 288 288 288 F F F 22 440-459 156 156 156 283 283 283 F F F 23 460-479 158 158 158 287 287 287 F F F 24 480-499 159 159 159 288 288 288 F F F 25 500-519 152 152 152 274 274 274 F F F 26 520-539 159 159 159 288 288 288 F F F 27 540-559 159 159 159 288 288 288 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 159, Max flush: 288 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.173520791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:01.216706491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.21714795Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.248858759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.249711891Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.249751836Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.249766404Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.249781422Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} {"msg":"Crucible 16704232-3815-406b-954f-a67b7a79ec2e has session id: 1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.250453599Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Upstairs opts: Upstairs UUID: 16704232-3815-406b-954f-a67b7a79ec2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.250495884Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Crucible stats registered with UUID: 16704232-3815-406b-954f-a67b7a79ec2e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.250513491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:01.250527039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.25103371Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251055447Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} {"msg":"16704232-3815-406b-954f-a67b7a79ec2e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251139965Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251157842Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"0","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251187469Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"0","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251202216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"1","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251221364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"1","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251235402Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"2","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251259228Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"2","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251282865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251313441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251342557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"ds_connection connected from Ok(127.0.0.1:53502)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251372723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"ds_connection connected from Ok(127.0.0.1:44203)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251404179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"ds_connection connected from Ok(127.0.0.1:43635)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.251504595Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.252289397Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"1","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.252334881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"2","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.252508257Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"0","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.252979122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"2","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253018157Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"1","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.25313723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"0","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.25357254Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253683415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[0]R flush_numbers[0..12]: [278, 267, 287, 288, 281, 286, 262, 267, 277, 277, 288, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.25379759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[0]R generation[0..12]: [154, 149, 158, 159, 156, 157, 145, 149, 154, 154, 159, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253819436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253835414Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[1]R flush_numbers[0..12]: [278, 267, 287, 288, 281, 286, 262, 267, 277, 277, 288, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253850202Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[1]R generation[0..12]: [154, 149, 158, 159, 156, 157, 145, 149, 154, 154, 159, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.25386461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253879118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[2]R flush_numbers[0..12]: [278, 267, 287, 288, 281, 286, 262, 267, 277, 277, 288, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253893226Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[2]R generation[0..12]: [154, 149, 158, 159, 156, 157, 145, 149, 154, 154, 159, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253907544Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253922032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253935551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253949079Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:13:01.253966296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.253986113Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"":"downstairs","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254000282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.25401298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"0","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254027068Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"1","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254040676Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"client":"2","":"downstairs","session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254054474Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254067012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"16704232-3815-406b-954f-a67b7a79ec2e is now active with session: 1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254079941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254092219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832,"session_id":"1ddf4ff8-37a6-4c6c-9e3b-4acc44277daf"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254104797Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} {"msg":"Activated sub_volume 16704232-3815-406b-954f-a67b7a79ec2e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:01.254116296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 16384 data: 18 15 12 12 02/30 Read block 429 len 4096 03/30 Write block 268 len 20480 data: 14 11 12 12 8 04/30 Read block 510 len 16384 05/30 Write block 270 len 8192 data: 13 13 06/30 Read block 576 len 4096 07/30 Flush 08/30 Write block 42 len 12288 data: 20 19 21 09/30 Write block 205 len 24576 data: 8 10 11 12 14 14 10/30 Write block 191 len 16384 data: 16 14 16 13 11/30 Read block 126 len 36864 12/30 Flush 13/30 Write block 9 len 4096 data: 9 14/30 Write block 119 len 20480 data: 16 15 17 18 15 15/30 Write block 21 len 24576 data: 9 8 9 9 8 8 16/30 Read block 329 len 8192 17/30 Read block 443 len 4096 18/30 Write block 326 len 36864 data: 8 8 9 12 14 14 12 14 12 19/30 Read block 344 len 16384 20/30 Write block 301 len 20480 data: 7 7 8 8 8 21/30 Read block 45 len 4096 22/30 Read block 79 len 20480 23/30 Read block 287 len 4096 24/30 Flush 25/30 Write block 538 len 16384 data: 15 17 14 13 26/30 Write block 290 len 32768 data: 11 8 8 6 6 8 7 8 27/30 Write block 417 len 24576 data: 10 12 11 9 11 13 28/30 Read block 526 len 24576 29/30 Read block 439 len 28672 30/30 Read block 592 len 8192 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:01.274026767Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1832} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 160 160 160 291 291 291 F F F 2 040-059 160 160 160 290 290 290 F F F 3 060-079 159 159 159 288 288 288 F F F 4 080-099 156 156 156 281 281 281 F F F 5 100-119 160 160 160 291 291 291 F F F 6 120-139 160 160 160 291 291 291 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 160 160 160 290 290 290 F F F 10 200-219 160 160 160 290 290 290 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 157 157 157 286 286 286 F F F 13 260-279 160 160 160 289 289 289 F F F 14 280-299 157 157 157 286 286 286 T T T 15 300-319 160 160 160 291 291 291 F F F 16 320-339 160 160 160 291 291 291 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 159 159 159 288 288 288 T T T 21 420-439 159 159 159 288 288 288 T T T 22 440-459 156 156 156 283 283 283 F F F 23 460-479 158 158 158 287 287 287 F F F 24 480-499 159 159 159 288 288 288 F F F 25 500-519 160 160 160 289 289 289 F F F 26 520-539 159 159 159 288 288 288 T T T 27 540-559 159 159 159 288 288 288 T T T 28 560-579 159 159 159 288 288 288 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 160, Max flush: 291 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 160 160 160 291 291 291 F F F 2 040-059 160 160 160 290 290 290 F F F 3 060-079 159 159 159 288 288 288 F F F 4 080-099 156 156 156 281 281 281 F F F 5 100-119 160 160 160 291 291 291 F F F 6 120-139 160 160 160 291 291 291 F F F 7 140-159 149 149 149 267 267 267 F F F 8 160-179 154 154 154 277 277 277 F F F 9 180-199 160 160 160 290 290 290 F F F 10 200-219 160 160 160 290 290 290 F F F 11 220-239 153 153 153 275 275 275 F F F 12 240-259 157 157 157 286 286 286 F F F 13 260-279 160 160 160 289 289 289 F F F 14 280-299 161 161 161 292 292 292 F F F 15 300-319 160 160 160 291 291 291 F F F 16 320-339 160 160 160 291 291 291 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 161 161 161 292 292 292 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 156 156 156 283 283 283 F F F 23 460-479 158 158 158 287 287 287 F F F 24 480-499 159 159 159 288 288 288 F F F 25 500-519 160 160 160 289 289 289 F F F 26 520-539 161 161 161 292 292 292 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 161, Max flush: 292 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.552667536Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.599084911Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.600003675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.60004311Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.600057987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.600072716Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} {"msg":"Crucible 34b71b57-42c3-4781-b5ae-bc514603bc11 has session id: bc3e870c-4e5d-48bc-b4cb-7a931b340309","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.600669933Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Upstairs opts: Upstairs UUID: 34b71b57-42c3-4781-b5ae-bc514603bc11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.600714627Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Crucible stats registered with UUID: 34b71b57-42c3-4781-b5ae-bc514603bc11","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.600738514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:05.600764141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601172294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601196791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} {"msg":"34b71b57-42c3-4781-b5ae-bc514603bc11 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601266071Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601284459Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"0","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601314125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"0","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601328843Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"1","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601343691Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"1","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601357609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601372497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601386215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601410881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601438638Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:05.601594006Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:05.601728808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"ds_connection connected from Ok(127.0.0.1:55671)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601759773Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"ds_connection connected from Ok(127.0.0.1:60689)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601791879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:05.601823545Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.601847352Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.60214805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"1","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.6022931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"0","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.602678907Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"1","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.602811729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"0","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.603336267Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:05.603465309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.603557752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"ds_connection connected from Ok(127.0.0.1:43822)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.603753305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.604705505Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.605303602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[0]R flush_numbers[0..12]: [291, 291, 290, 288, 281, 291, 291, 267, 277, 290, 290, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.605970151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[0]R generation[0..12]: [160, 160, 160, 159, 156, 160, 160, 149, 154, 160, 160, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606018274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606041501Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[1]R flush_numbers[0..12]: [291, 291, 290, 288, 281, 291, 291, 267, 277, 290, 290, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606060578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[1]R generation[0..12]: [160, 160, 160, 159, 156, 160, 160, 149, 154, 160, 160, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606078656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606096933Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[2]R flush_numbers[0..12]: [291, 291, 290, 288, 281, 291, 291, 267, 277, 290, 290, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606114731Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[2]R generation[0..12]: [160, 160, 160, 159, 156, 160, 160, 149, 154, 160, 160, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606132518Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606150576Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606167333Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606184441Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606201299Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606218346Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"":"downstairs","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606235314Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606250902Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"0","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606270119Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"1","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606287527Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"client":"2","":"downstairs","session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606310534Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606326562Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"34b71b57-42c3-4781-b5ae-bc514603bc11 is now active with session: bc3e870c-4e5d-48bc-b4cb-7a931b340309","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606343019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606358387Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840,"session_id":"bc3e870c-4e5d-48bc-b4cb-7a931b340309"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606374205Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} {"msg":"Activated sub_volume 34b71b57-42c3-4781-b5ae-bc514603bc11","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:15.606388823Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 196 len 12288 data: 14 10 13 02/30 Write block 59 len 16384 data: 16 16 19 21 03/30 Write block 190 len 40960 data: 13 17 15 17 14 12 15 11 14 13 04/30 Write block 204 len 24576 data: 10 9 11 12 13 15 05/30 Read block 272 len 40960 06/30 Flush 07/30 Write block 224 len 24576 data: 13 13 14 14 16 14 08/30 Read block 23 len 40960 09/30 Write block 288 len 24576 data: 13 11 12 9 9 7 10/30 Write block 528 len 20480 data: 15 18 21 19 19 11/30 Write block 367 len 28672 data: 12 12 14 14 13 12 10 12/30 Write block 440 len 12288 data: 10 13 12 13/30 Write block 520 len 4096 data: 14 14/30 Read block 75 len 28672 15/30 Read block 288 len 40960 16/30 Read block 104 len 8192 17/30 Write block 152 len 36864 data: 13 12 17 18 21 19 18 15 15 18/30 Read block 198 len 32768 19/30 Write block 512 len 40960 data: 17 18 20 20 17 15 15 11 15 14 20/30 Read block 354 len 24576 21/30 Read block 445 len 36864 22/30 Read block 210 len 24576 23/30 Read block 266 len 36864 24/30 Read block 53 len 4096 25/30 Write block 259 len 8192 data: 21 20 26/30 Write block 525 len 12288 data: 11 14 13 27/30 Read block 217 len 20480 28/30 Read block 337 len 4096 29/30 Write block 451 len 28672 data: 22 19 16 13 12 11 13 30/30 Read block 58 len 16384 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 24 22 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:15.62691356Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1840} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 160 160 160 291 291 291 F F F 2 040-059 162 162 162 293 293 293 F F F 3 060-079 162 162 162 293 293 293 F F F 4 080-099 156 156 156 281 281 281 F F F 5 100-119 160 160 160 291 291 291 F F F 6 120-139 160 160 160 291 291 291 F F F 7 140-159 149 149 149 267 267 267 T T T 8 160-179 154 154 154 277 277 277 T T T 9 180-199 162 162 162 293 293 293 F F F 10 200-219 162 162 162 293 293 293 F F F 11 220-239 153 153 153 275 275 275 T T T 12 240-259 157 157 157 286 286 286 T T T 13 260-279 160 160 160 289 289 289 T T T 14 280-299 161 161 161 292 292 292 T T T 15 300-319 160 160 160 291 291 291 F F F 16 320-339 160 160 160 291 291 291 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 153 153 153 275 275 275 T T T 19 380-399 158 158 158 287 287 287 F F F 20 400-419 161 161 161 292 292 292 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 156 156 156 283 283 283 T T T 23 460-479 158 158 158 287 287 287 F F F 24 480-499 159 159 159 288 288 288 F F F 25 500-519 160 160 160 289 289 289 T T T 26 520-539 161 161 161 292 292 292 T T T 27 540-559 161 161 161 292 292 292 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 162, Max flush: 293 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 160 160 160 291 291 291 F F F 2 040-059 162 162 162 293 293 293 F F F 3 060-079 162 162 162 293 293 293 F F F 4 080-099 156 156 156 281 281 281 F F F 5 100-119 160 160 160 291 291 291 F F F 6 120-139 160 160 160 291 291 291 F F F 7 140-159 163 163 163 294 294 294 F F F 8 160-179 163 163 163 294 294 294 F F F 9 180-199 162 162 162 293 293 293 F F F 10 200-219 162 162 162 293 293 293 F F F 11 220-239 163 163 163 294 294 294 F F F 12 240-259 163 163 163 294 294 294 F F F 13 260-279 163 163 163 294 294 294 F F F 14 280-299 163 163 163 294 294 294 F F F 15 300-319 160 160 160 291 291 291 F F F 16 320-339 160 160 160 291 291 291 F F F 17 340-359 155 155 155 279 279 279 F F F 18 360-379 163 163 163 294 294 294 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 161 161 161 292 292 292 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 163 163 163 294 294 294 F F F 23 460-479 158 158 158 287 287 287 F F F 24 480-499 159 159 159 288 288 288 F F F 25 500-519 163 163 163 294 294 294 F F F 26 520-539 163 163 163 294 294 294 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 163, Max flush: 294 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.898491876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:29.941436439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.941831375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.973120141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.973834413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.973879097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.973896604Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.973915022Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} {"msg":"Crucible f812e543-7d49-44be-b9c0-ef2d27d0f1db has session id: 85dd837b-7e9c-44e0-949a-068e3dd4e9e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.974532967Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Upstairs opts: Upstairs UUID: f812e543-7d49-44be-b9c0-ef2d27d0f1db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.974577921Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Crucible stats registered with UUID: f812e543-7d49-44be-b9c0-ef2d27d0f1db","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.974606177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:29.974624724Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975087981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975110877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} {"msg":"f812e543-7d49-44be-b9c0-ef2d27d0f1db active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975197126Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975216683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"0","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975248598Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"0","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975265216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"1","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975280514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"1","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975294442Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"2","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.97530955Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"2","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975324128Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975339076Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975353634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41089)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975605159Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36758)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975669901Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60796)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.975703106Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.976527512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"2","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.976563318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"1","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.976668483Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"0","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.97712519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"1","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.977235035Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"2","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.977382155Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"0","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.97777858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.977872547Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[0]R flush_numbers[0..12]: [291, 291, 293, 293, 281, 291, 291, 294, 294, 293, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.977983862Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[0]R generation[0..12]: [160, 160, 162, 162, 156, 160, 160, 163, 163, 162, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.97800364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978019047Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[1]R flush_numbers[0..12]: [291, 291, 293, 293, 281, 291, 291, 294, 294, 293, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978033575Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[1]R generation[0..12]: [160, 160, 162, 162, 156, 160, 160, 163, 163, 162, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978051443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978066881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[2]R flush_numbers[0..12]: [291, 291, 293, 293, 281, 291, 291, 294, 294, 293, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978081439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[2]R generation[0..12]: [160, 160, 162, 162, 156, 160, 160, 163, 163, 162, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978095747Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978110315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978123893Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978137631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978151279Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978164787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"":"downstairs","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978178485Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978191014Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"0","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978204862Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"1","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978225089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"client":"2","":"downstairs","session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978239017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978252105Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"f812e543-7d49-44be-b9c0-ef2d27d0f1db is now active with session: 85dd837b-7e9c-44e0-949a-068e3dd4e9e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978267243Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978279802Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848,"session_id":"85dd837b-7e9c-44e0-949a-068e3dd4e9e5"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.97829247Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} {"msg":"Activated sub_volume f812e543-7d49-44be-b9c0-ef2d27d0f1db","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:29.978304418Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 197 len 4096 data: 12 02/30 Flush 03/30 Write block 349 len 40960 data: 13 14 16 16 16 16 12 14 12 10 04/30 Read block 588 len 40960 05/30 Read block 510 len 16384 06/30 Write block 159 len 28672 data: 16 16 14 13 12 13 13 07/30 Read block 47 len 8192 08/30 Write block 304 len 8192 data: 9 9 09/30 Read block 555 len 8192 10/30 Flush 11/30 Write block 74 len 4096 data: 7 12/30 Read block 478 len 4096 13/30 Write block 96 len 12288 data: 17 14 9 14/30 Flush 15/30 Write block 117 len 36864 data: 13 12 17 16 18 19 16 16 19 16/30 Read block 499 len 4096 17/30 Read block 130 len 20480 18/30 Read block 187 len 4096 19/30 Flush 20/30 Write block 24 len 24576 data: 10 9 9 6 6 6 21/30 Read block 140 len 16384 22/30 Read block 122 len 32768 23/30 Write block 38 len 4096 data: 6 24/30 Write block 499 len 8192 data: 18 18 25/30 Read block 162 len 12288 26/30 Write block 333 len 24576 data: 15 13 15 18 17 19 27/30 Write block 302 len 24576 data: 8 9 10 10 7 7 28/30 Read block 125 len 4096 29/30 Read block 233 len 8192 30/30 Read block 361 len 16384 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 10 11 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:29.998260293Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1848} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 160 160 160 291 291 291 T T T 2 040-059 162 162 162 293 293 293 F F F 3 060-079 164 164 164 297 297 297 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 164 164 164 298 298 298 F F F 6 120-139 164 164 164 298 298 298 F F F 7 140-159 164 164 164 296 296 296 F F F 8 160-179 164 164 164 296 296 296 F F F 9 180-199 164 164 164 295 295 295 F F F 10 200-219 162 162 162 293 293 293 F F F 11 220-239 163 163 163 294 294 294 F F F 12 240-259 163 163 163 294 294 294 F F F 13 260-279 163 163 163 294 294 294 F F F 14 280-299 163 163 163 294 294 294 F F F 15 300-319 164 164 164 296 296 296 T T T 16 320-339 160 160 160 291 291 291 T T T 17 340-359 164 164 164 296 296 296 F F F 18 360-379 163 163 163 294 294 294 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 161 161 161 292 292 292 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 163 163 163 294 294 294 F F F 23 460-479 158 158 158 287 287 287 F F F 24 480-499 159 159 159 288 288 288 T T T 25 500-519 163 163 163 294 294 294 T T T 26 520-539 163 163 163 294 294 294 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 164, Max flush: 298 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 165 165 165 299 299 299 F F F 2 040-059 162 162 162 293 293 293 F F F 3 060-079 164 164 164 297 297 297 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 164 164 164 298 298 298 F F F 6 120-139 164 164 164 298 298 298 F F F 7 140-159 164 164 164 296 296 296 F F F 8 160-179 164 164 164 296 296 296 F F F 9 180-199 164 164 164 295 295 295 F F F 10 200-219 162 162 162 293 293 293 F F F 11 220-239 163 163 163 294 294 294 F F F 12 240-259 163 163 163 294 294 294 F F F 13 260-279 163 163 163 294 294 294 F F F 14 280-299 163 163 163 294 294 294 F F F 15 300-319 165 165 165 299 299 299 F F F 16 320-339 165 165 165 299 299 299 F F F 17 340-359 164 164 164 296 296 296 F F F 18 360-379 163 163 163 294 294 294 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 161 161 161 292 292 292 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 163 163 163 294 294 294 F F F 23 460-479 158 158 158 287 287 287 F F F 24 480-499 165 165 165 299 299 299 F F F 25 500-519 165 165 165 299 299 299 F F F 26 520-539 163 163 163 294 294 294 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 154 154 154 277 277 277 F F F Max gen: 165, Max flush: 299 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.283003393Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.327900857Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.328677071Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.328747461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.328764829Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.328779967Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} {"msg":"Crucible 3dd42310-019a-46fd-b7ff-17e913e357d4 has session id: e16611d1-2e88-4ef0-aa7e-1bb7030970bc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.329309854Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Upstairs opts: Upstairs UUID: 3dd42310-019a-46fd-b7ff-17e913e357d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.329368115Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Crucible stats registered with UUID: 3dd42310-019a-46fd-b7ff-17e913e357d4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.329391822Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:34.329444335Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.329966193Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.329997469Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} {"msg":"3dd42310-019a-46fd-b7ff-17e913e357d4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330095335Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330115723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"0","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330148388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"0","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330163976Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330179264Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330199611Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"2","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330222718Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"2","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330248385Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.33028099Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330312946Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:50072)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330466055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54592)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330525496Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:34.330765723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:34.330810547Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:34.330843963Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.330922692Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.331292821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"2","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.331322357Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"0","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.332044167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"0","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.332087381Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"2","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.332737752Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:34.332768248Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.332833816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:61356)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.33294882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.333997856Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.334497297Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[0]R flush_numbers[0..12]: [291, 299, 293, 297, 297, 298, 298, 296, 296, 295, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335095355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[0]R generation[0..12]: [160, 165, 162, 164, 164, 164, 164, 164, 164, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335118162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335134239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[1]R flush_numbers[0..12]: [291, 299, 293, 297, 297, 298, 298, 296, 296, 295, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335148827Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[1]R generation[0..12]: [160, 165, 162, 164, 164, 164, 164, 164, 164, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335163035Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335177753Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[2]R flush_numbers[0..12]: [291, 299, 293, 297, 297, 298, 298, 296, 296, 295, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335198311Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[2]R generation[0..12]: [160, 165, 162, 164, 164, 164, 164, 164, 164, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335213278Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335227807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335241274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335254933Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335268481Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335281949Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"":"downstairs","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335295617Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335308005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"0","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335321903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"1","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335335372Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"client":"2","":"downstairs","session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Wait for a query_work_queue command to finish before sending IO All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335354059Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335366877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"3dd42310-019a-46fd-b7ff-17e913e357d4 is now active with session: e16611d1-2e88-4ef0-aa7e-1bb7030970bc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335380195Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335392664Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856,"session_id":"e16611d1-2e88-4ef0-aa7e-1bb7030970bc"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.335405182Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} {"msg":"Activated sub_volume 3dd42310-019a-46fd-b7ff-17e913e357d4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:44.33541873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 459 len 36864 data: 11 6 9 7 6 8 10 12 9 02/30 Flush 03/30 Write block 103 len 32768 data: 13 16 14 12 15 15 15 11 04/30 Read block 195 len 32768 05/30 Flush 06/30 Write block 281 len 12288 data: 15 15 14 07/30 Flush 08/30 Write block 149 len 28672 data: 14 14 13 14 13 18 19 09/30 Read block 409 len 8192 10/30 Flush 11/30 Write block 43 len 32768 data: 20 22 19 19 18 15 11 13 12/30 Flush 13/30 Write block 582 len 32768 data: 8 8 9 12 11 12 12 14 14/30 Write block 529 len 24576 data: 19 22 20 20 19 18 15/30 Write block 294 len 16384 data: 7 9 8 9 16/30 Read block 99 len 8192 17/30 Write block 41 len 40960 data: 16 21 21 23 20 20 19 16 12 14 18/30 Read block 255 len 4096 19/30 Read block 379 len 32768 20/30 Write block 246 len 32768 data: 18 18 19 16 17 12 10 10 21/30 Flush 22/30 Flush 23/30 Write block 48 len 4096 data: 17 24/30 Read block 106 len 24576 25/30 Read block 170 len 28672 26/30 Write block 510 len 16384 data: 14 15 18 19 27/30 Write block 465 len 28672 data: 11 13 10 11 11 12 15 28/30 Write block 482 len 16384 data: 11 14 18 17 29/30 Read block 415 len 28672 30/30 Read block 316 len 36864 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:44.357515201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 165 165 165 299 299 299 F F F 2 040-059 166 166 166 305 305 305 T T T 3 060-079 164 164 164 297 297 297 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 298 298 298 F F F 7 140-159 166 166 166 303 303 303 F F F 8 160-179 164 164 164 296 296 296 F F F 9 180-199 164 164 164 295 295 295 F F F 10 200-219 162 162 162 293 293 293 F F F 11 220-239 163 163 163 294 294 294 F F F 12 240-259 166 166 166 305 305 305 F F F 13 260-279 163 163 163 294 294 294 F F F 14 280-299 166 166 166 305 305 305 F F F 15 300-319 165 165 165 299 299 299 F F F 16 320-339 165 165 165 299 299 299 F F F 17 340-359 164 164 164 296 296 296 F F F 18 360-379 163 163 163 294 294 294 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 161 161 161 292 292 292 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 166 166 166 300 300 300 F F F 23 460-479 166 166 166 300 300 300 T T T 24 480-499 165 165 165 299 299 299 T T T 25 500-519 165 165 165 299 299 299 T T T 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 166, Max flush: 305 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 F F F 1 020-039 165 165 165 299 299 299 F F F 2 040-059 167 167 167 306 306 306 F F F 3 060-079 164 164 164 297 297 297 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 298 298 298 F F F 7 140-159 166 166 166 303 303 303 F F F 8 160-179 164 164 164 296 296 296 F F F 9 180-199 164 164 164 295 295 295 F F F 10 200-219 162 162 162 293 293 293 F F F 11 220-239 163 163 163 294 294 294 F F F 12 240-259 166 166 166 305 305 305 F F F 13 260-279 163 163 163 294 294 294 F F F 14 280-299 166 166 166 305 305 305 F F F 15 300-319 165 165 165 299 299 299 F F F 16 320-339 165 165 165 299 299 299 F F F 17 340-359 164 164 164 296 296 296 F F F 18 360-379 163 163 163 294 294 294 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 161 161 161 292 292 292 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 166 166 166 300 300 300 F F F 23 460-479 167 167 167 306 306 306 F F F 24 480-499 167 167 167 306 306 306 F F F 25 500-519 167 167 167 306 306 306 F F F 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 159 159 159 288 288 288 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 167, Max flush: 306 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.595762706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.639910303Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.640650821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.640688116Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.640702914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.640723251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} {"msg":"Crucible 0ff85353-773a-4a73-b139-80088f40b949 has session id: 96a68f7f-c0b3-4b23-bf30-3087676bca84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.64131602Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Upstairs opts: Upstairs UUID: 0ff85353-773a-4a73-b139-80088f40b949, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.641362734Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Crucible stats registered with UUID: 0ff85353-773a-4a73-b139-80088f40b949","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.64138613Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:48.641407557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.641802453Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.641828419Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} {"msg":"0ff85353-773a-4a73-b139-80088f40b949 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.641903239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.641922117Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"0","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.641955142Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"0","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.64197142Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"1","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.641986918Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"1","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.642018193Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"2","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.6420428Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"2","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.642081165Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.642110991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.642138217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"ds_connection connected from Ok(127.0.0.1:55986)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.642296305Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"ds_connection connected from Ok(127.0.0.1:56090)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.642341459Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"ds_connection connected from Ok(127.0.0.1:55236)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.642373364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.64298482Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"0","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.643025675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"1","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.643688304Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"1","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.643726298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"0","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.644380688Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.644417943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.644890708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"2","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.64538789Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"2","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[0]R flush_numbers[0..12]: [291, 299, 306, 297, 297, 301, 298, 303, 296, 295, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.645891921Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[0]R generation[0..12]: [160, 165, 167, 164, 164, 166, 164, 166, 164, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.645916367Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.645931745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[1]R flush_numbers[0..12]: [291, 299, 306, 297, 297, 301, 298, 303, 296, 295, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.645946043Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[1]R generation[0..12]: [160, 165, 167, 164, 164, 166, 164, 166, 164, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.645960181Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.645975659Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[2]R flush_numbers[0..12]: [291, 299, 306, 297, 297, 301, 298, 303, 296, 295, 293, 294]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.645990287Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[2]R generation[0..12]: [160, 165, 167, 164, 164, 166, 164, 166, 164, 164, 162, 163]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646032431Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646047789Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646061317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646075125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646095473Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646108851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"":"downstairs","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646122739Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646135017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"0","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646153655Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"1","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646167873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"client":"2","":"downstairs","session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646181641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646194349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"0ff85353-773a-4a73-b139-80088f40b949 is now active with session: 96a68f7f-c0b3-4b23-bf30-3087676bca84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646207537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646219795Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864,"session_id":"96a68f7f-c0b3-4b23-bf30-3087676bca84"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646232364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} {"msg":"Activated sub_volume 0ff85353-773a-4a73-b139-80088f40b949","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:13:48.646243722Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 570 len 32768 data: 12 13 12 10 10 11 8 9 02/30 Read block 357 len 16384 03/30 Write block 454 len 36864 data: 14 13 12 14 15 12 7 10 8 04/30 Read block 535 len 40960 05/30 Write block 248 len 4096 data: 20 06/30 Read block 461 len 20480 07/30 Read block 351 len 16384 08/30 Read block 329 len 36864 09/30 Read block 161 len 20480 10/30 Read block 493 len 8192 11/30 Read block 457 len 8192 12/30 Write block 486 len 20480 data: 16 15 13 14 15 13/30 Read block 389 len 4096 14/30 Write block 219 len 32768 data: 10 12 12 13 13 14 14 15 15/30 Write block 365 len 12288 data: 9 10 13 16/30 Read block 449 len 20480 17/30 Write block 402 len 8192 data: 11 14 18/30 Write block 198 len 36864 data: 15 14 12 13 12 11 11 10 12 19/30 Read block 115 len 32768 20/30 Read block 326 len 4096 21/30 Write block 465 len 36864 data: 12 14 11 12 12 13 16 14 15 22/30 Flush 23/30 Write block 2 len 36864 data: 5 6 6 6 8 7 7 10 13 24/30 Write block 333 len 16384 data: 16 14 16 19 25/30 Write block 229 len 12288 data: 15 14 13 26/30 Read block 263 len 32768 27/30 Write block 230 len 20480 data: 15 14 15 13 11 28/30 Read block 112 len 20480 29/30 Write block 356 len 32768 data: 15 13 11 9 7 7 6 7 30/30 Read block 178 len 40960 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 6 8 8 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:13:48.667889085Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1864} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 291 291 291 T T T 1 020-039 165 165 165 299 299 299 F F F 2 040-059 167 167 167 306 306 306 F F F 3 060-079 164 164 164 297 297 297 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 298 298 298 F F F 7 140-159 166 166 166 303 303 303 F F F 8 160-179 164 164 164 296 296 296 F F F 9 180-199 168 168 168 307 307 307 F F F 10 200-219 168 168 168 307 307 307 F F F 11 220-239 168 168 168 307 307 307 T T T 12 240-259 168 168 168 307 307 307 F F F 13 260-279 163 163 163 294 294 294 F F F 14 280-299 166 166 166 305 305 305 F F F 15 300-319 165 165 165 299 299 299 F F F 16 320-339 165 165 165 299 299 299 T T T 17 340-359 164 164 164 296 296 296 T T T 18 360-379 168 168 168 307 307 307 T T T 19 380-399 158 158 158 287 287 287 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 168 168 168 307 307 307 F F F 23 460-479 168 168 168 307 307 307 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 167 167 167 306 306 306 F F F 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 168, Max flush: 307 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 308 308 308 F F F 1 020-039 165 165 165 299 299 299 F F F 2 040-059 167 167 167 306 306 306 F F F 3 060-079 164 164 164 297 297 297 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 166 166 166 301 301 301 F F F 6 120-139 164 164 164 298 298 298 F F F 7 140-159 166 166 166 303 303 303 F F F 8 160-179 164 164 164 296 296 296 F F F 9 180-199 168 168 168 307 307 307 F F F 10 200-219 168 168 168 307 307 307 F F F 11 220-239 169 169 169 308 308 308 F F F 12 240-259 168 168 168 307 307 307 F F F 13 260-279 163 163 163 294 294 294 F F F 14 280-299 166 166 166 305 305 305 F F F 15 300-319 165 165 165 299 299 299 F F F 16 320-339 169 169 169 308 308 308 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 158 158 158 287 287 287 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 168 168 168 307 307 307 F F F 23 460-479 168 168 168 307 307 307 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 167 167 167 306 306 306 F F F 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 169, Max flush: 308 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.917056683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:02.96164609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.962212712Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.994373609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.995063114Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.995098449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.995112567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.995127196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} {"msg":"Crucible 9456a767-78fb-4a6a-ba56-69ded80c1309 has session id: a19b19be-4be0-4835-94de-ab9edf002094","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.995742171Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Upstairs opts: Upstairs UUID: 9456a767-78fb-4a6a-ba56-69ded80c1309, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.995775876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Crucible stats registered with UUID: 9456a767-78fb-4a6a-ba56-69ded80c1309","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.995791344Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:02.995804502Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996236833Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996268708Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} {"msg":"9456a767-78fb-4a6a-ba56-69ded80c1309 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.9963326Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996350237Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"0","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996379393Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"0","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996394301Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"1","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996409419Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"1","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996423257Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"2","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996440395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"2","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996459122Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996479929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996496117Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"ds_connection connected from Ok(127.0.0.1:57343)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996594294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"ds_connection connected from Ok(127.0.0.1:35820)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996790077Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"ds_connection connected from Ok(127.0.0.1:56767)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.996824832Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.997306246Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"2","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.997470803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"1","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.997506958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"0","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.997941978Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"2","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.997972434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"1","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998120334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"0","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998575571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998610986Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[0]R flush_numbers[0..12]: [308, 299, 306, 297, 297, 301, 298, 303, 296, 307, 307, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998735269Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[0]R generation[0..12]: [169, 165, 167, 164, 164, 166, 164, 166, 164, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998764055Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998780303Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[1]R flush_numbers[0..12]: [308, 299, 306, 297, 297, 301, 298, 303, 296, 307, 307, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998795131Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[1]R generation[0..12]: [169, 165, 167, 164, 164, 166, 164, 166, 164, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998810009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998824637Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[2]R flush_numbers[0..12]: [308, 299, 306, 297, 297, 301, 298, 303, 296, 307, 307, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998839385Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[2]R generation[0..12]: [169, 165, 167, 164, 164, 166, 164, 166, 164, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998856772Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998871661Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998884999Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998903616Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998927403Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998958769Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"":"downstairs","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.998984325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.999006152Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"0","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.99902336Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"1","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.999044826Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"client":"2","":"downstairs","session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.999070003Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.99908936Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"9456a767-78fb-4a6a-ba56-69ded80c1309 is now active with session: a19b19be-4be0-4835-94de-ab9edf002094","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.999103218Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.999115557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872,"session_id":"a19b19be-4be0-4835-94de-ab9edf002094"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.999128035Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} {"msg":"Activated sub_volume 9456a767-78fb-4a6a-ba56-69ded80c1309","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:02.999139553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 133 len 40960 data: 13 13 15 11 11 9 7 8 7 7 02/30 Flush 03/30 Flush 04/30 Flush 05/30 Write block 66 len 28672 data: 16 17 15 15 13 12 10 06/30 Write block 189 len 36864 data: 13 14 18 16 18 15 13 16 13 07/30 Read block 582 len 16384 08/30 Write block 198 len 12288 data: 16 15 13 09/30 Read block 198 len 28672 10/30 Read block 566 len 8192 11/30 Read block 331 len 28672 12/30 Read block 116 len 8192 13/30 Write block 381 len 32768 data: 13 14 12 10 11 11 10 12 14/30 Write block 241 len 4096 data: 19 15/30 Read block 561 len 4096 16/30 Read block 432 len 40960 17/30 Flush 18/30 Write block 118 len 32768 data: 13 18 17 19 20 17 17 20 19/30 Read block 193 len 36864 20/30 Write block 295 len 32768 data: 10 9 10 8 9 7 8 9 21/30 Read block 583 len 32768 22/30 Flush 23/30 Write block 275 len 32768 data: 9 9 10 11 13 15 16 16 24/30 Write block 167 len 24576 data: 9 12 14 14 14 14 25/30 Write block 161 len 4096 data: 15 26/30 Write block 101 len 20480 data: 11 10 14 17 15 27/30 Write block 201 len 12288 data: 14 13 12 28/30 Write block 109 len 4096 data: 16 29/30 Write block 323 len 20480 data: 10 10 10 9 9 30/30 Write block 15 len 20480 data: 10 8 10 12 14 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:03.019502583Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1872} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 308 308 308 T T T 1 020-039 165 165 165 299 299 299 F F F 2 040-059 167 167 167 306 306 306 F F F 3 060-079 170 170 170 312 312 312 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 170 170 170 313 313 313 T T T 6 120-139 170 170 170 313 313 313 F F F 7 140-159 170 170 170 309 309 309 F F F 8 160-179 164 164 164 296 296 296 T T T 9 180-199 170 170 170 312 312 312 F F F 10 200-219 170 170 170 312 312 312 T T T 11 220-239 169 169 169 308 308 308 F F F 12 240-259 170 170 170 312 312 312 F F F 13 260-279 163 163 163 294 294 294 T T T 14 280-299 170 170 170 313 313 313 T T T 15 300-319 170 170 170 313 313 313 F F F 16 320-339 169 169 169 308 308 308 T T T 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 168 168 168 307 307 307 F F F 23 460-479 168 168 168 307 307 307 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 167 167 167 306 306 306 F F F 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 170, Max flush: 313 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 314 314 314 F F F 1 020-039 165 165 165 299 299 299 F F F 2 040-059 167 167 167 306 306 306 F F F 3 060-079 170 170 170 312 312 312 F F F 4 080-099 164 164 164 297 297 297 F F F 5 100-119 171 171 171 314 314 314 F F F 6 120-139 170 170 170 313 313 313 F F F 7 140-159 170 170 170 309 309 309 F F F 8 160-179 171 171 171 314 314 314 F F F 9 180-199 170 170 170 312 312 312 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 169 169 169 308 308 308 F F F 12 240-259 170 170 170 312 312 312 F F F 13 260-279 171 171 171 314 314 314 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 171 171 171 314 314 314 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 168 168 168 307 307 307 F F F 23 460-479 168 168 168 307 307 307 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 167 167 167 306 306 306 F F F 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 168 168 168 307 307 307 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 171, Max flush: 314 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.305962977Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.350338893Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.351166129Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.351200705Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.351215343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.35123417Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} {"msg":"Crucible 9bed418e-61bd-4be0-b5eb-450b08325edf has session id: 865c9974-593e-427d-8ff5-5de7e9192e95","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.351882641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Upstairs opts: Upstairs UUID: 9bed418e-61bd-4be0-b5eb-450b08325edf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.351927555Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Crucible stats registered with UUID: 9bed418e-61bd-4be0-b5eb-450b08325edf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.351953561Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:07.351976258Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352511934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352536821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} {"msg":"9bed418e-61bd-4be0-b5eb-450b08325edf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.35261465Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352632358Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"0","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352661214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"0","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352675592Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"1","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.35269033Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"1","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352704068Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"2","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352718646Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"2","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352732204Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352750031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352765329Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"ds_connection connected from Ok(127.0.0.1:52761)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.352902941Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"ds_connection connected from Ok(127.0.0.1:62866)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.353176663Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"ds_connection connected from Ok(127.0.0.1:35023)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.353215618Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.353606514Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"2","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.353746774Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"0","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.35414519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"2","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.354317746Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"0","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.354760765Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.354885478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.355119886Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"1","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.355612168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"1","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[0]R flush_numbers[0..12]: [314, 299, 306, 312, 297, 314, 313, 309, 314, 312, 314, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356112089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[0]R generation[0..12]: [171, 165, 167, 170, 164, 171, 170, 170, 171, 170, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356140315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356157773Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[1]R flush_numbers[0..12]: [314, 299, 306, 312, 297, 314, 313, 309, 314, 312, 314, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356172731Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[1]R generation[0..12]: [171, 165, 167, 170, 164, 171, 170, 170, 171, 170, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356187229Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356201537Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[2]R flush_numbers[0..12]: [314, 299, 306, 312, 297, 314, 313, 309, 314, 312, 314, 308]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356215745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[2]R generation[0..12]: [171, 165, 167, 170, 164, 171, 170, 170, 171, 170, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356229933Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356244541Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356258039Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356271797Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356285115Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356298623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"":"downstairs","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:14:07.356312572Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356330979Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"0","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356346787Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"1","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356360785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"client":"2","":"downstairs","session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356374583Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356387141Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"9bed418e-61bd-4be0-b5eb-450b08325edf is now active with session: 865c9974-593e-427d-8ff5-5de7e9192e95","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.35640016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356412498Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880,"session_id":"865c9974-593e-427d-8ff5-5de7e9192e95"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356425326Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} {"msg":"Activated sub_volume 9bed418e-61bd-4be0-b5eb-450b08325edf","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:07.356436884Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 568 len 36864 data: 15 13 13 14 13 11 11 12 9 02/30 Write block 440 len 40960 data: 11 14 13 14 17 18 19 23 23 23 03/30 Write block 564 len 20480 data: 17 17 16 14 16 04/30 Write block 73 len 28672 data: 12 8 8 13 13 13 12 05/30 Read block 81 len 8192 06/30 Write block 80 len 20480 data: 12 14 11 12 14 07/30 Read block 127 len 28672 08/30 Write block 9 len 28672 data: 11 14 11 12 13 13 11 09/30 Read block 183 len 36864 10/30 Write block 509 len 32768 data: 11 15 16 19 20 21 21 18 11/30 Write block 124 len 24576 data: 18 21 18 14 13 11 12/30 Read block 388 len 4096 13/30 Read block 587 len 40960 14/30 Write block 459 len 24576 data: 13 8 11 9 7 9 15/30 Read block 522 len 16384 16/30 Read block 183 len 8192 17/30 Write block 79 len 40960 data: 13 13 15 12 13 15 12 16 12 14 18/30 Write block 227 len 40960 data: 15 17 16 16 15 16 14 12 12 17 19/30 Read block 540 len 40960 20/30 Read block 224 len 4096 21/30 Read block 188 len 28672 22/30 Read block 191 len 24576 23/30 Read block 528 len 20480 24/30 Flush 25/30 Write block 35 len 32768 data: 4 6 5 7 10 12 17 22 26/30 Write block 245 len 4096 data: 19 27/30 Read block 259 len 32768 28/30 Read block 303 len 16384 29/30 Write block 236 len 40960 data: 18 19 16 17 18 20 16 17 17 20 30/30 Write block 320 len 28672 data: 9 11 10 11 11 11 10 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:07.378243135Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1880} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 315 315 315 F F F 1 020-039 165 165 165 299 299 299 T T T 2 040-059 167 167 167 306 306 306 T T T 3 060-079 172 172 172 315 315 315 F F F 4 080-099 172 172 172 315 315 315 F F F 5 100-119 171 171 171 314 314 314 F F F 6 120-139 172 172 172 315 315 315 F F F 7 140-159 170 170 170 309 309 309 F F F 8 160-179 171 171 171 314 314 314 F F F 9 180-199 170 170 170 312 312 312 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 172 172 172 315 315 315 T T T 12 240-259 170 170 170 312 312 312 T T T 13 260-279 171 171 171 314 314 314 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 171 171 171 314 314 314 T T T 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 172 172 172 315 315 315 F F F 23 460-479 172 172 172 315 315 315 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 172 172 172 315 315 315 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 172, Max flush: 315 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 315 315 315 F F F 1 020-039 173 173 173 316 316 316 F F F 2 040-059 173 173 173 316 316 316 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 172 172 172 315 315 315 F F F 5 100-119 171 171 171 314 314 314 F F F 6 120-139 172 172 172 315 315 315 F F F 7 140-159 170 170 170 309 309 309 F F F 8 160-179 171 171 171 314 314 314 F F F 9 180-199 170 170 170 312 312 312 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 173 173 173 316 316 316 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 171 171 171 314 314 314 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 173 173 173 316 316 316 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 172 172 172 315 315 315 F F F 23 460-479 172 172 172 315 315 315 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 166 166 166 305 305 305 F F F 27 540-559 161 161 161 292 292 292 F F F 28 560-579 172 172 172 315 315 315 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 173, Max flush: 316 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.627823873Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.672290437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.673119313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.673167266Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.673186293Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.673206121Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} {"msg":"Crucible bd93026f-85c7-4940-8094-b00764f908ff has session id: faf8a695-0977-4db3-bc92-a800c4b9f266","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.673724979Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Upstairs opts: Upstairs UUID: bd93026f-85c7-4940-8094-b00764f908ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.673777022Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Crucible stats registered with UUID: bd93026f-85c7-4940-8094-b00764f908ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.673804439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:11.673820816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674307359Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674342285Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} {"msg":"bd93026f-85c7-4940-8094-b00764f908ff active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.67437396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674388658Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"0","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674421803Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"0","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674438251Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"1","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674463858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"1","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674490704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"2","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.67451962Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"2","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674547806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674576392Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674614577Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"ds_connection connected from Ok(127.0.0.1:50455)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674644053Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"ds_connection connected from Ok(127.0.0.1:34070)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674672559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"ds_connection connected from Ok(127.0.0.1:59657)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.674701925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.675366004Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"0","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.675397549Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"1","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.67612131Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"0","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.676167663Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"1","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.676925579Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.676958715Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.677318535Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"2","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.677849062Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"2","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[0]R flush_numbers[0..12]: [315, 316, 316, 315, 315, 314, 315, 309, 314, 312, 314, 316]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678381349Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[0]R generation[0..12]: [172, 173, 173, 172, 172, 171, 172, 170, 171, 170, 171, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678406295Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678425143Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[1]R flush_numbers[0..12]: [315, 316, 316, 315, 315, 314, 315, 309, 314, 312, 314, 316]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678443341Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[1]R generation[0..12]: [172, 173, 173, 172, 172, 171, 172, 170, 171, 170, 171, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678461208Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678479096Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[2]R flush_numbers[0..12]: [315, 316, 316, 315, 315, 314, 315, 309, 314, 312, 314, 316]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678496853Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[2]R generation[0..12]: [172, 173, 173, 172, 172, 171, 172, 170, 171, 170, 171, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678514551Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678532638Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678559625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678577422Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.67859434Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678611227Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"":"downstairs","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678628175Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678643403Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"0","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678660651Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"1","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678677528Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"client":"2","":"downstairs","session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678694676Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678710334Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 bd93026f-85c7-4940-8094-b00764f908ff is now active with session: faf8a695-0977-4db3-bc92-a800c4b9f266","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678729001Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678744919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888,"session_id":"faf8a695-0977-4db3-bc92-a800c4b9f266"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678760857Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} {"msg":"Activated sub_volume bd93026f-85c7-4940-8094-b00764f908ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:11.678775315Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 542 len 32768 data: 13 12 9 11 9 10 11 12 03/30 Write block 463 len 8192 data: 8 10 04/30 Write block 559 len 24576 data: 12 13 15 16 16 18 05/30 Read block 0 len 16384 06/30 Read block 523 len 20480 07/30 Write block 228 len 4096 data: 18 08/30 Read block 550 len 20480 09/30 Write block 324 len 4096 data: 12 10/30 Read block 469 len 20480 11/30 Flush 12/30 Write block 7 len 32768 data: 8 8 12 15 12 13 14 14 13/30 Write block 444 len 16384 data: 18 19 20 24 14/30 Flush 15/30 Write block 541 len 12288 data: 14 14 13 16/30 Write block 533 len 28672 data: 20 19 16 16 17 16 18 17/30 Read block 92 len 20480 18/30 Read block 369 len 36864 19/30 Read block 15 len 40960 20/30 Write block 108 len 24576 data: 16 17 12 11 9 9 21/30 Write block 98 len 32768 data: 10 12 12 12 11 15 18 16 22/30 Write block 456 len 4096 data: 13 23/30 Read block 4 len 12288 24/30 Read block 513 len 8192 25/30 Read block 191 len 16384 26/30 Read block 241 len 16384 27/30 Read block 115 len 36864 28/30 Read block 304 len 28672 29/30 Read block 429 len 32768 30/30 Write block 25 len 12288 data: 10 10 7 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 3 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:11.699763118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1888} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 319 319 319 F F F 1 020-039 173 173 173 316 316 316 T T T 2 040-059 173 173 173 316 316 316 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 172 172 172 315 315 315 T T T 5 100-119 171 171 171 314 314 314 T T T 6 120-139 172 172 172 315 315 315 F F F 7 140-159 170 170 170 309 309 309 F F F 8 160-179 171 171 171 314 314 314 F F F 9 180-199 170 170 170 312 312 312 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 174 174 174 318 318 318 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 171 171 171 314 314 314 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 174 174 174 318 318 318 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 174 174 174 319 319 319 T T T 23 460-479 174 174 174 318 318 318 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 166 166 166 305 305 305 T T T 27 540-559 174 174 174 318 318 318 T T T 28 560-579 174 174 174 318 318 318 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 174, Max flush: 319 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 319 319 319 F F F 1 020-039 175 175 175 320 320 320 F F F 2 040-059 173 173 173 316 316 316 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 175 175 175 320 320 320 F F F 5 100-119 175 175 175 320 320 320 F F F 6 120-139 172 172 172 315 315 315 F F F 7 140-159 170 170 170 309 309 309 F F F 8 160-179 171 171 171 314 314 314 F F F 9 180-199 170 170 170 312 312 312 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 174 174 174 318 318 318 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 171 171 171 314 314 314 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 174 174 174 318 318 318 F F F 17 340-359 169 169 169 308 308 308 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 161 161 161 292 292 292 F F F 22 440-459 175 175 175 320 320 320 F F F 23 460-479 174 174 174 318 318 318 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 174 174 174 318 318 318 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 175, Max flush: 320 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.953597559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.998409886Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.999192328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.999233932Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.99925085Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.999269148Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} {"msg":"Crucible f753ac87-7ecf-4a2a-a3ad-a7db4e785054 has session id: 61ff1a5c-82ca-4417-b730-a1d9e9aeddd4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.999821461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Upstairs opts: Upstairs UUID: f753ac87-7ecf-4a2a-a3ad-a7db4e785054, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.999860706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Crucible stats registered with UUID: f753ac87-7ecf-4a2a-a3ad-a7db4e785054","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:15.999877104Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:15.999890362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000355628Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000383894Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} {"msg":"f753ac87-7ecf-4a2a-a3ad-a7db4e785054 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.00048981Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000507597Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"0","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000537043Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"0","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.00056216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"1","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000590146Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"1","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000617092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000647628Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000676644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.00070568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.000741745Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:16.000776Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:16.000825763Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33623)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.00099282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56828)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.001037914Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:16.00106702Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.001082238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.001486053Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"0","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.001527957Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"1","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.002207543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"0","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.002245168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"1","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.002765597Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:16.002850885Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.002141437Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35812)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.002219206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.003516428Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.00408264Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[0]R flush_numbers[0..12]: [319, 320, 316, 315, 320, 320, 315, 309, 314, 312, 314, 318]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004659031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[0]R generation[0..12]: [174, 175, 173, 172, 175, 175, 172, 170, 171, 170, 171, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004691116Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004711263Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[1]R flush_numbers[0..12]: [319, 320, 316, 315, 320, 320, 315, 309, 314, 312, 314, 318]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004730261Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[1]R generation[0..12]: [174, 175, 173, 172, 175, 175, 172, 170, 171, 170, 171, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004748798Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004767656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[2]R flush_numbers[0..12]: [319, 320, 316, 315, 320, 320, 315, 309, 314, 312, 314, 318]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004785543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[2]R generation[0..12]: [174, 175, 173, 172, 175, 175, 172, 170, 171, 170, 171, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004803631Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004822058Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004838916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004862203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.00487975Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004896898Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"":"downstairs","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"No downstairs reconciliation required","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004918865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004934673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"0","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.00495211Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"1","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004969338Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"client":"2","":"downstairs","session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.004986386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.005002203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"f753ac87-7ecf-4a2a-a3ad-a7db4e785054 is now active with session: 61ff1a5c-82ca-4417-b730-a1d9e9aeddd4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.005018881Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.005034439Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896,"session_id":"61ff1a5c-82ca-4417-b730-a1d9e9aeddd4"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.005050427Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} {"msg":"Activated sub_volume f753ac87-7ecf-4a2a-a3ad-a7db4e785054","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:26.005064915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 36864 data: 15 11 10 9 10 11 11 11 8 02/30 Read block 174 len 16384 03/30 Write block 6 len 24576 data: 9 9 9 13 16 13 04/30 Write block 348 len 28672 data: 14 14 15 17 17 17 17 05/30 Flush 06/30 Write block 434 len 4096 data: 11 07/30 Write block 562 len 24576 data: 17 17 19 18 17 15 08/30 Read block 359 len 8192 09/30 Write block 133 len 36864 data: 14 14 16 12 12 10 8 9 8 10/30 Flush 11/30 Write block 141 len 20480 data: 9 8 9 10 13 12/30 Write block 179 len 16384 data: 13 16 19 17 13/30 Write block 445 len 40960 data: 20 21 25 24 24 21 23 20 17 15 14/30 Read block 199 len 28672 15/30 Read block 46 len 20480 16/30 Read block 577 len 8192 17/30 Flush 18/30 Write block 14 len 32768 data: 15 12 9 11 13 16 12 11 19/30 Read block 492 len 32768 20/30 Write block 267 len 36864 data: 14 15 12 14 14 9 9 7 10 21/30 Read block 435 len 40960 22/30 Write block 183 len 8192 data: 14 12 23/30 Read block 15 len 4096 24/30 Read block 172 len 20480 25/30 Write block 1 len 4096 data: 2 26/30 Read block 206 len 40960 27/30 Read block 322 len 20480 28/30 Write block 48 len 16384 data: 18 13 15 14 29/30 Read block 553 len 28672 30/30 Read block 354 len 20480 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:26.026717477Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1896} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 321 321 321 T T T 1 020-039 176 176 176 321 321 321 T T T 2 040-059 173 173 173 316 316 316 T T T 3 060-079 172 172 172 315 315 315 F F F 4 080-099 175 175 175 320 320 320 F F F 5 100-119 175 175 175 320 320 320 F F F 6 120-139 176 176 176 322 322 322 F F F 7 140-159 176 176 176 323 323 323 F F F 8 160-179 176 176 176 323 323 323 F F F 9 180-199 176 176 176 323 323 323 T T T 10 200-219 171 171 171 314 314 314 F F F 11 220-239 174 174 174 318 318 318 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 171 171 171 314 314 314 T T T 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 174 174 174 318 318 318 F F F 17 340-359 176 176 176 321 321 321 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 176 176 176 322 322 322 F F F 22 440-459 176 176 176 323 323 323 F F F 23 460-479 174 174 174 318 318 318 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 176, Max flush: 323 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 324 324 324 F F F 1 020-039 177 177 177 324 324 324 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 175 175 175 320 320 320 F F F 5 100-119 175 175 175 320 320 320 F F F 6 120-139 176 176 176 322 322 322 F F F 7 140-159 176 176 176 323 323 323 F F F 8 160-179 176 176 176 323 323 323 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 171 171 171 314 314 314 F F F 11 220-239 174 174 174 318 318 318 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 177 177 177 324 324 324 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 174 174 174 318 318 318 F F F 17 340-359 176 176 176 321 321 321 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 168 168 168 307 307 307 F F F 21 420-439 176 176 176 322 322 322 F F F 22 440-459 176 176 176 323 323 323 F F F 23 460-479 174 174 174 318 318 318 F F F 24 480-499 168 168 168 307 307 307 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 166 166 166 305 305 305 F F F Max gen: 177, Max flush: 324 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.27058313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.315309468Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.316008692Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.316047906Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.316063084Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.316077762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} {"msg":"Crucible 029ffab2-fa23-46cc-adb2-380f1c4a64ca has session id: b56602ab-752f-4f1f-9c80-212a20bae2ee","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.316735312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Upstairs opts: Upstairs UUID: 029ffab2-fa23-46cc-adb2-380f1c4a64ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.316778956Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Crucible stats registered with UUID: 029ffab2-fa23-46cc-adb2-380f1c4a64ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.316804022Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:30.316818201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317384943Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317416418Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} {"msg":"029ffab2-fa23-46cc-adb2-380f1c4a64ca active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317495507Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317515635Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"0","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317554679Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"0","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317582006Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"1","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317611571Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"1","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317644137Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"2","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317677622Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"2","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317707188Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317737724Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.31776976Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:61776)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317927808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:59894)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.317976481Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:43415)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.318118882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.318652268Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"0","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.318681584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"2","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.319377009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"2","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.319493783Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"0","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.319598278Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"1","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.319954409Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320122386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320283234Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"1","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 324, 315, 320, 320, 322, 323, 323, 324, 314, 318]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320830729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[0]R generation[0..12]: [177, 177, 177, 172, 175, 175, 176, 176, 176, 177, 171, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320852136Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320867184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 324, 315, 320, 320, 322, 323, 323, 324, 314, 318]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320881532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[1]R generation[0..12]: [177, 177, 177, 172, 175, 175, 176, 176, 176, 177, 171, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320898489Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320913177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 324, 315, 320, 320, 322, 323, 323, 324, 314, 318]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320927475Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[2]R generation[0..12]: [177, 177, 177, 172, 175, 175, 176, 176, 176, 177, 171, 174]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320941593Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320955951Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320971479Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.320988167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321004685Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321021333Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"":"downstairs","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.32103816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2025-03-18T05:14:30.321062607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"0","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321080134Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"1","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321097272Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"client":"2","":"downstairs","session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.32111415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321133207Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"029ffab2-fa23-46cc-adb2-380f1c4a64ca is now active with session: b56602ab-752f-4f1f-9c80-212a20bae2ee","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321150005Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321165273Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904,"session_id":"b56602ab-752f-4f1f-9c80-212a20bae2ee"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321180911Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} {"msg":"Activated sub_volume 029ffab2-fa23-46cc-adb2-380f1c4a64ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:30.321195009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 127 len 28672 data: 15 14 12 10 11 11 15 02/30 Write block 237 len 16384 data: 20 17 18 19 03/30 Read block 331 len 32768 04/30 Read block 2 len 24576 05/30 Write block 332 len 40960 data: 13 17 15 17 20 18 20 18 17 13 06/30 Flush 07/30 Write block 487 len 24576 data: 16 14 15 16 14 11 08/30 Read block 573 len 24576 09/30 Read block 457 len 12288 10/30 Read block 548 len 12288 11/30 Read block 490 len 40960 12/30 Read block 9 len 4096 13/30 Read block 175 len 28672 14/30 Read block 424 len 4096 15/30 Read block 275 len 24576 16/30 Read block 459 len 36864 17/30 Read block 553 len 12288 18/30 Write block 400 len 36864 data: 10 10 12 15 13 11 10 13 12 19/30 Read block 542 len 24576 20/30 Read block 108 len 8192 21/30 Read block 481 len 40960 22/30 Flush 23/30 Write block 161 len 28672 data: 16 14 13 14 14 7 10 24/30 Write block 215 len 4096 data: 15 25/30 Write block 591 len 28672 data: 14 16 17 17 14 13 12 26/30 Read block 139 len 8192 27/30 Read block 17 len 36864 28/30 Write block 138 len 28672 data: 11 9 10 10 9 10 11 29/30 Read block 533 len 4096 30/30 Read block 166 len 40960 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:30.344258966Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1904} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 324 324 324 F F F 1 020-039 177 177 177 324 324 324 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 175 175 175 320 320 320 F F F 5 100-119 175 175 175 320 320 320 F F F 6 120-139 178 178 178 325 325 325 T T T 7 140-159 176 176 176 323 323 323 T T T 8 160-179 176 176 176 323 323 323 T T T 9 180-199 177 177 177 324 324 324 F F F 10 200-219 171 171 171 314 314 314 T T T 11 220-239 178 178 178 325 325 325 F F F 12 240-259 178 178 178 325 325 325 F F F 13 260-279 177 177 177 324 324 324 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 178 178 178 325 325 325 F F F 17 340-359 178 178 178 325 325 325 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 178 178 178 326 326 326 F F F 21 420-439 176 176 176 322 322 322 F F F 22 440-459 176 176 176 323 323 323 F F F 23 460-479 174 174 174 318 318 318 F F F 24 480-499 178 178 178 326 326 326 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 166 166 166 305 305 305 T T T Max gen: 178, Max flush: 326 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 324 324 324 F F F 1 020-039 177 177 177 324 324 324 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 175 175 175 320 320 320 F F F 5 100-119 175 175 175 320 320 320 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 179 179 179 327 327 327 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 179 179 179 327 327 327 F F F 11 220-239 178 178 178 325 325 325 F F F 12 240-259 178 178 178 325 325 325 F F F 13 260-279 177 177 177 324 324 324 F F F 14 280-299 171 171 171 314 314 314 F F F 15 300-319 170 170 170 313 313 313 F F F 16 320-339 178 178 178 325 325 325 F F F 17 340-359 178 178 178 325 325 325 F F F 18 360-379 169 169 169 308 308 308 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 178 178 178 326 326 326 F F F 21 420-439 176 176 176 322 322 322 F F F 22 440-459 176 176 176 323 323 323 F F F 23 460-479 174 174 174 318 318 318 F F F 24 480-499 178 178 178 326 326 326 F F F 25 500-519 172 172 172 315 315 315 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 179 179 179 327 327 327 F F F Max gen: 179, Max flush: 327 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.593576519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.640690089Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.641722877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.641767111Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.641782389Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.641797007Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} {"msg":"Crucible 9f1de36b-4a2c-42ec-a41d-e447ff13cc64 has session id: f4765604-bd25-4808-90b9-aeb20da243db","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.642351101Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Upstairs opts: Upstairs UUID: 9f1de36b-4a2c-42ec-a41d-e447ff13cc64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.642393645Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Crucible stats registered with UUID: 9f1de36b-4a2c-42ec-a41d-e447ff13cc64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.642415422Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:34.642438319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.642984393Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643091099Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} {"msg":"9f1de36b-4a2c-42ec-a41d-e447ff13cc64 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643122585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643146711Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"0","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643212932Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"0","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643242928Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"1","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643335985Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"1","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643368771Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"2","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643546196Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"2","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643581581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643604798Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643632044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"ds_connection connected from Ok(127.0.0.1:60938)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643659331Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"ds_connection connected from Ok(127.0.0.1:36172)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643682917Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"ds_connection connected from Ok(127.0.0.1:44617)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.643706184Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.644138125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"0","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.64417399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"2","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.644199606Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"1","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.644764059Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"0","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.644846248Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"1","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.644867405Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"2","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645354807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.64548134Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 324, 315, 320, 320, 327, 327, 327, 324, 327, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645606023Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[0]R generation[0..12]: [177, 177, 177, 172, 175, 175, 179, 179, 179, 177, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.64562726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645642348Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 324, 315, 320, 320, 327, 327, 327, 324, 327, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645657016Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[1]R generation[0..12]: [177, 177, 177, 172, 175, 175, 179, 179, 179, 177, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645671354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645685642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 324, 315, 320, 320, 327, 327, 327, 324, 327, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.64569971Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[2]R generation[0..12]: [177, 177, 177, 172, 175, 175, 179, 179, 179, 177, 179, 178]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645713988Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645728386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645741704Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645755103Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645768321Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:14:34.645785238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"":"downstairs","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645803286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645815894Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"0","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645829512Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"1","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.64584293Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"client":"2","":"downstairs","session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645856438Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645868977Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"9f1de36b-4a2c-42ec-a41d-e447ff13cc64 is now active with session: f4765604-bd25-4808-90b9-aeb20da243db","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645881995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645894183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912,"session_id":"f4765604-bd25-4808-90b9-aeb20da243db"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.645906562Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} {"msg":"Activated sub_volume 9f1de36b-4a2c-42ec-a41d-e447ff13cc64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:34.64591793Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 12288 data: 7 8 7 02/30 Read block 67 len 12288 03/30 Write block 303 len 32768 data: 10 11 11 8 8 9 7 8 04/30 Read block 588 len 16384 05/30 Read block 299 len 16384 06/30 Read block 8 len 8192 07/30 Read block 244 len 28672 08/30 Read block 294 len 8192 09/30 Read block 344 len 40960 10/30 Read block 304 len 28672 11/30 Write block 294 len 24576 data: 8 11 10 11 9 10 12/30 Flush 13/30 Flush 14/30 Write block 264 len 32768 data: 16 17 16 15 16 13 15 15 15/30 Flush 16/30 Write block 60 len 20480 data: 17 20 22 21 18 17/30 Flush 18/30 Write block 80 len 20480 data: 14 16 13 14 16 19/30 Write block 597 len 8192 data: 13 11 20/30 Read block 250 len 24576 21/30 Read block 516 len 24576 22/30 Write block 347 len 28672 data: 14 15 15 16 18 18 18 23/30 Read block 439 len 24576 24/30 Read block 587 len 32768 25/30 Write block 19 len 36864 data: 17 13 12 10 11 12 12 12 9 26/30 Write block 227 len 36864 data: 16 19 17 17 16 17 15 13 13 27/30 Write block 503 len 16384 data: 19 16 13 13 28/30 Write block 409 len 28672 data: 13 14 12 14 16 16 16 29/30 Write block 406 len 20480 data: 11 14 13 14 15 30/30 Read block 31 len 28672 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:34.666761253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1912} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 324 324 324 T T T 1 020-039 177 177 177 324 324 324 T T T 2 040-059 177 177 177 324 324 324 F F F 3 060-079 180 180 180 331 331 331 F F F 4 080-099 175 175 175 320 320 320 T T T 5 100-119 175 175 175 320 320 320 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 179 179 179 327 327 327 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 179 179 179 327 327 327 F F F 11 220-239 178 178 178 325 325 325 T T T 12 240-259 178 178 178 325 325 325 F F F 13 260-279 180 180 180 330 330 330 F F F 14 280-299 180 180 180 328 328 328 F F F 15 300-319 180 180 180 328 328 328 F F F 16 320-339 178 178 178 325 325 325 F F F 17 340-359 178 178 178 325 325 325 T T T 18 360-379 180 180 180 328 328 328 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 178 178 178 326 326 326 T T T 21 420-439 176 176 176 322 322 322 F F F 22 440-459 176 176 176 323 323 323 F F F 23 460-479 174 174 174 318 318 318 F F F 24 480-499 178 178 178 326 326 326 F F F 25 500-519 172 172 172 315 315 315 T T T 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 179 179 179 327 327 327 T T T Max gen: 180, Max flush: 331 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 332 332 332 F F F 1 020-039 181 181 181 332 332 332 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 180 180 180 331 331 331 F F F 4 080-099 181 181 181 332 332 332 F F F 5 100-119 175 175 175 320 320 320 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 179 179 179 327 327 327 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 179 179 179 327 327 327 F F F 11 220-239 181 181 181 332 332 332 F F F 12 240-259 178 178 178 325 325 325 F F F 13 260-279 180 180 180 330 330 330 F F F 14 280-299 180 180 180 328 328 328 F F F 15 300-319 180 180 180 328 328 328 F F F 16 320-339 178 178 178 325 325 325 F F F 17 340-359 181 181 181 332 332 332 F F F 18 360-379 180 180 180 328 328 328 F F F 19 380-399 170 170 170 312 312 312 F F F 20 400-419 181 181 181 332 332 332 F F F 21 420-439 176 176 176 322 322 322 F F F 22 440-459 176 176 176 323 323 323 F F F 23 460-479 174 174 174 318 318 318 F F F 24 480-499 178 178 178 326 326 326 F F F 25 500-519 181 181 181 332 332 332 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 181 181 181 332 332 332 F F F Max gen: 181, Max flush: 332 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:38.935438373Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:38.978549584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:38.978947689Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.010617343Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.011281471Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.011316567Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.011331125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.011345543Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} {"msg":"Crucible 82ff43cc-456a-4371-9f25-0bcc80166d07 has session id: 235982f8-50cd-4776-8a94-325de04f4754","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.011885648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Upstairs opts: Upstairs UUID: 82ff43cc-456a-4371-9f25-0bcc80166d07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.011922573Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Crucible stats registered with UUID: 82ff43cc-456a-4371-9f25-0bcc80166d07","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.011953919Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:39.011977916Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012353804Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012379031Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} {"msg":"82ff43cc-456a-4371-9f25-0bcc80166d07 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.01245316Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012472318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"0","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012504683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"0","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.01253077Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"1","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012560126Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"1","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012592411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"2","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012620807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"2","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012648443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012680339Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012712995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"ds_connection connected from Ok(127.0.0.1:47451)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012780535Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"ds_connection connected from Ok(127.0.0.1:33144)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012836478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"ds_connection connected from Ok(127.0.0.1:33993)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.012951222Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.013429396Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"1","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.013611041Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"2","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.013809994Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"0","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014020555Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"1","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014237225Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"2","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014330312Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"0","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014613883Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014679064Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[0]R flush_numbers[0..12]: [332, 332, 324, 331, 332, 320, 327, 327, 327, 324, 327, 332]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014909103Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[0]R generation[0..12]: [181, 181, 177, 180, 181, 175, 179, 179, 179, 177, 179, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014931169Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014946858Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[1]R flush_numbers[0..12]: [332, 332, 324, 331, 332, 320, 327, 327, 327, 324, 327, 332]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014961385Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[1]R generation[0..12]: [181, 181, 177, 180, 181, 175, 179, 179, 179, 177, 179, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014980793Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.014995961Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[2]R flush_numbers[0..12]: [332, 332, 324, 331, 332, 320, 327, 327, 327, 324, 327, 332]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015010319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[2]R generation[0..12]: [181, 181, 177, 180, 181, 175, 179, 179, 179, 177, 179, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015024557Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015038995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015052593Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015066421Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO "downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.01514764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015173556Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"":"downstairs","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015188634Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015201642Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"0","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015215741Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"1","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015229699Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"client":"2","":"downstairs","session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015243806Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015262214Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"82ff43cc-456a-4371-9f25-0bcc80166d07 is now active with session: 235982f8-50cd-4776-8a94-325de04f4754","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015276262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.01528889Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920,"session_id":"235982f8-50cd-4776-8a94-325de04f4754"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015301679Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} {"msg":"Activated sub_volume 82ff43cc-456a-4371-9f25-0bcc80166d07","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:39.015313327Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 20 len 12288 data: 14 13 11 02/30 Read block 288 len 20480 03/30 Read block 394 len 8192 04/30 Read block 63 len 32768 05/30 Write block 442 len 12288 data: 14 15 19 06/30 Write block 510 len 24576 data: 16 17 20 21 22 22 07/30 Read block 445 len 12288 08/30 Read block 297 len 40960 09/30 Write block 380 len 32768 data: 13 14 15 13 11 12 12 11 10/30 Read block 67 len 32768 11/30 Write block 259 len 8192 data: 22 21 12/30 Read block 528 len 28672 13/30 Write block 88 len 16384 data: 15 14 15 16 14/30 Read block 300 len 32768 15/30 Read block 547 len 36864 16/30 Read block 372 len 8192 17/30 Read block 221 len 12288 18/30 Read block 187 len 20480 19/30 Write block 172 len 12288 data: 15 14 12 20/30 Read block 155 len 28672 21/30 Read block 303 len 24576 22/30 Write block 298 len 28672 data: 10 11 8 9 10 11 12 23/30 Write block 482 len 12288 data: 12 15 19 24/30 Read block 168 len 8192 25/30 Read block 151 len 12288 26/30 Read block 175 len 32768 27/30 Write block 460 len 16384 data: 9 12 10 9 28/30 Write block 201 len 28672 data: 15 14 13 12 11 13 13 29/30 Read block 192 len 12288 30/30 Read block 467 len 32768 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:39.036988976Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1920} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 332 332 332 F F F 1 020-039 181 181 181 332 332 332 T T T 2 040-059 177 177 177 324 324 324 F F F 3 060-079 180 180 180 331 331 331 F F F 4 080-099 181 181 181 332 332 332 T T T 5 100-119 175 175 175 320 320 320 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 179 179 179 327 327 327 T T T 9 180-199 177 177 177 324 324 324 F F F 10 200-219 179 179 179 327 327 327 T T T 11 220-239 181 181 181 332 332 332 F F F 12 240-259 178 178 178 325 325 325 T T T 13 260-279 180 180 180 330 330 330 T T T 14 280-299 180 180 180 328 328 328 T T T 15 300-319 180 180 180 328 328 328 T T T 16 320-339 178 178 178 325 325 325 F F F 17 340-359 181 181 181 332 332 332 F F F 18 360-379 180 180 180 328 328 328 F F F 19 380-399 170 170 170 312 312 312 T T T 20 400-419 181 181 181 332 332 332 F F F 21 420-439 176 176 176 322 322 322 F F F 22 440-459 176 176 176 323 323 323 T T T 23 460-479 174 174 174 318 318 318 T T T 24 480-499 178 178 178 326 326 326 T T T 25 500-519 181 181 181 332 332 332 T T T 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 181 181 181 332 332 332 F F F Max gen: 181, Max flush: 332 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 332 332 332 F F F 1 020-039 182 182 182 333 333 333 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 180 180 180 331 331 331 F F F 4 080-099 182 182 182 333 333 333 F F F 5 100-119 175 175 175 320 320 320 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 182 182 182 333 333 333 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 181 181 181 332 332 332 F F F 12 240-259 182 182 182 333 333 333 F F F 13 260-279 182 182 182 333 333 333 F F F 14 280-299 182 182 182 333 333 333 F F F 15 300-319 182 182 182 333 333 333 F F F 16 320-339 178 178 178 325 325 325 F F F 17 340-359 181 181 181 332 332 332 F F F 18 360-379 180 180 180 328 328 328 F F F 19 380-399 182 182 182 333 333 333 F F F 20 400-419 181 181 181 332 332 332 F F F 21 420-439 176 176 176 322 322 322 F F F 22 440-459 182 182 182 333 333 333 F F F 23 460-479 182 182 182 333 333 333 F F F 24 480-499 182 182 182 333 333 333 F F F 25 500-519 182 182 182 333 333 333 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 F F F 29 580-599 181 181 181 332 332 332 F F F Max gen: 182, Max flush: 333 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.355896797Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:43.400228369Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.400699295Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.43279837Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.433434012Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.433469048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.433482995Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.433497824Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} {"msg":"Crucible ac78b206-d4b5-4fa9-ad4b-69c2cf2942ea has session id: 56f3ea1d-d07f-45ad-867e-8edf004b50f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.4341056Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Upstairs opts: Upstairs UUID: ac78b206-d4b5-4fa9-ad4b-69c2cf2942ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434153113Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Crucible stats registered with UUID: ac78b206-d4b5-4fa9-ad4b-69c2cf2942ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.43418019Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:43.434195118Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.43468568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.43476038Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} {"msg":"ac78b206-d4b5-4fa9-ad4b-69c2cf2942ea active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434783897Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434798015Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"0","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434826401Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"0","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434846958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"1","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434873644Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"1","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434900341Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"2","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434929547Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"2","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434955773Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.434983979Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.435015505Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59172)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.435320393Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52395)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.435375625Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53215)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.435397172Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.435971663Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"1","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.436008898Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"2","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.436145729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"0","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.436670497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"1","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.436699043Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"2","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.436762735Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"0","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437277044Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437390888Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[0]R flush_numbers[0..12]: [332, 333, 324, 331, 333, 320, 327, 327, 333, 324, 333, 332]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437456609Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[0]R generation[0..12]: [181, 182, 177, 180, 182, 175, 179, 179, 182, 177, 182, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437484535Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437501133Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[1]R flush_numbers[0..12]: [332, 333, 324, 331, 333, 320, 327, 327, 333, 324, 333, 332]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437515811Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[1]R generation[0..12]: [181, 182, 177, 180, 182, 175, 179, 179, 182, 177, 182, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437530029Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437544517Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[2]R flush_numbers[0..12]: [332, 333, 324, 331, 333, 320, 327, 327, 333, 324, 333, 332]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437562865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[2]R generation[0..12]: [181, 182, 177, 180, 182, 175, 179, 179, 182, 177, 182, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437577403Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.43759532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Max found gen is 183","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2025-03-18T05:14:43.437626186Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Generation requested: 184 >= found:183","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437648023Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.43766899Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437683308Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"":"downstairs","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437696956Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437709364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"0","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437731871Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"1","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437752808Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"client":"2","":"downstairs","session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437766846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437779425Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"ac78b206-d4b5-4fa9-ad4b-69c2cf2942ea is now active with session: 56f3ea1d-d07f-45ad-867e-8edf004b50f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437792683Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.4378136Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928,"session_id":"56f3ea1d-d07f-45ad-867e-8edf004b50f0"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:14:43.437832947Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} {"msg":"Activated sub_volume ac78b206-d4b5-4fa9-ad4b-69c2cf2942ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:43.437851895Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 40960 data: 8 9 12 11 12 11 12 9 10 11 02/30 Write block 589 len 20480 data: 15 16 15 17 18 03/30 Write block 417 len 36864 data: 11 13 12 10 12 14 13 13 15 04/30 Flush 05/30 Write block 104 len 32768 data: 19 17 13 16 17 18 13 12 06/30 Read block 54 len 12288 07/30 Write block 464 len 12288 data: 11 13 15 08/30 Write block 221 len 20480 data: 13 14 14 15 15 09/30 Read block 338 len 20480 10/30 Write block 590 len 20480 data: 17 16 18 19 18 11/30 Read block 118 len 16384 12/30 Read block 557 len 28672 13/30 Read block 76 len 28672 14/30 Read block 144 len 4096 15/30 Write block 471 len 24576 data: 17 15 16 11 11 8 16/30 Write block 5 len 24576 data: 7 10 10 10 14 17 17/30 Write block 241 len 40960 data: 21 17 18 18 21 19 19 21 17 18 18/30 Read block 541 len 8192 19/30 Write block 459 len 16384 data: 14 10 13 11 20/30 Read block 254 len 20480 21/30 Read block 411 len 28672 22/30 Read block 72 len 12288 23/30 Read block 229 len 40960 24/30 Write block 577 len 40960 data: 10 13 14 11 9 9 9 10 13 12 25/30 Read block 294 len 4096 26/30 Read block 81 len 24576 27/30 Write block 443 len 20480 data: 16 20 21 22 26 28/30 Read block 62 len 36864 29/30 Read block 321 len 16384 30/30 Read block 496 len 32768 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 8 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 26 26 25 77 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:43.460343861Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 332 332 332 T T T 1 020-039 182 182 182 333 333 333 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 180 180 180 331 331 331 F F F 4 080-099 182 182 182 333 333 333 F F F 5 100-119 175 175 175 320 320 320 T T T 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 182 182 182 333 333 333 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 181 181 181 332 332 332 T T T 12 240-259 182 182 182 333 333 333 T T T 13 260-279 182 182 182 333 333 333 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 184 184 184 334 334 334 F F F 16 320-339 178 178 178 325 325 325 F F F 17 340-359 181 181 181 332 332 332 F F F 18 360-379 180 180 180 328 328 328 F F F 19 380-399 182 182 182 333 333 333 F F F 20 400-419 184 184 184 334 334 334 F F F 21 420-439 184 184 184 334 334 334 F F F 22 440-459 182 182 182 333 333 333 T T T 23 460-479 182 182 182 333 333 333 T T T 24 480-499 182 182 182 333 333 333 F F F 25 500-519 182 182 182 333 333 333 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 176 176 176 322 322 322 T T T 29 580-599 184 184 184 334 334 334 T T T Max gen: 184, Max flush: 334 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 335 335 335 F F F 1 020-039 182 182 182 333 333 333 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 180 180 180 331 331 331 F F F 4 080-099 182 182 182 333 333 333 F F F 5 100-119 185 185 185 335 335 335 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 182 182 182 333 333 333 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 185 185 185 335 335 335 F F F 12 240-259 185 185 185 335 335 335 F F F 13 260-279 182 182 182 333 333 333 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 184 184 184 334 334 334 F F F 16 320-339 178 178 178 325 325 325 F F F 17 340-359 181 181 181 332 332 332 F F F 18 360-379 180 180 180 328 328 328 F F F 19 380-399 182 182 182 333 333 333 F F F 20 400-419 184 184 184 334 334 334 F F F 21 420-439 184 184 184 334 334 334 F F F 22 440-459 185 185 185 335 335 335 F F F 23 460-479 185 185 185 335 335 335 F F F 24 480-499 182 182 182 333 333 333 F F F 25 500-519 182 182 182 333 333 333 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 175 175 175 320 320 320 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 185 185 185 335 335 335 F F F Max gen: 185, Max flush: 335 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.753810962Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:47.797147791Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.797640813Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.829159968Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.829806559Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.829842124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.829856952Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.82987162Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} {"msg":"Crucible 1cc876cc-1d71-490e-9d5b-39cf80dce5bb has session id: a2f3eed1-0dc3-4990-9eb1-b5c8b7789146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.830542408Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Upstairs opts: Upstairs UUID: 1cc876cc-1d71-490e-9d5b-39cf80dce5bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.830586062Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Crucible stats registered with UUID: 1cc876cc-1d71-490e-9d5b-39cf80dce5bb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.830606219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:47.830622937Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831001105Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.83103436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} {"msg":"1cc876cc-1d71-490e-9d5b-39cf80dce5bb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831090982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831118588Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"0","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831152684Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"0","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831168682Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"1","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831183799Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"1","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831197627Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"2","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831212216Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"2","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831225763Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831240401Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831261419Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"ds_connection connected from Ok(127.0.0.1:63529)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831371723Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"ds_connection connected from Ok(127.0.0.1:43519)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831409698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"ds_connection connected from Ok(127.0.0.1:59755)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.831439234Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.832183772Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"1","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.832214527Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"2","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.832333371Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"0","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.832858969Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"1","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.832898913Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"2","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.832929779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"0","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833500481Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833526507Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[0]R flush_numbers[0..12]: [335, 333, 324, 331, 333, 335, 327, 327, 333, 324, 333, 335]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833593328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[0]R generation[0..12]: [185, 182, 177, 180, 182, 185, 179, 179, 182, 177, 182, 185]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833612805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833628353Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[1]R flush_numbers[0..12]: [335, 333, 324, 331, 333, 335, 327, 327, 333, 324, 333, 335]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833642671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[1]R generation[0..12]: [185, 182, 177, 180, 182, 185, 179, 179, 182, 177, 182, 185]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833656929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833671607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[2]R flush_numbers[0..12]: [335, 333, 324, 331, 333, 335, 327, 327, 333, 324, 333, 335]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833685785Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[2]R generation[0..12]: [185, 182, 177, 180, 182, 185, 179, 179, 182, 177, 182, 185]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833699903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833714231Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:14:47.833730879Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833813388Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833832565Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833846903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"":"downstairs","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833860851Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833873359Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"0","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833887268Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"1","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833901386Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"client":"2","":"downstairs","session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833915444Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833928302Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"1cc876cc-1d71-490e-9d5b-39cf80dce5bb is now active with session: a2f3eed1-0dc3-4990-9eb1-b5c8b7789146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.83394298Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833955678Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936,"session_id":"a2f3eed1-0dc3-4990-9eb1-b5c8b7789146"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833968276Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} {"msg":"Activated sub_volume 1cc876cc-1d71-490e-9d5b-39cf80dce5bb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:47.833979825Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 16384 data: 15 12 11 13 02/30 Write block 332 len 32768 data: 14 18 16 18 21 19 21 19 03/30 Read block 557 len 4096 04/30 Write block 9 len 20480 data: 15 18 14 14 15 05/30 Write block 208 len 8192 data: 14 16 06/30 Read block 31 len 24576 07/30 Read block 598 len 4096 08/30 Read block 49 len 40960 09/30 Write block 545 len 32768 data: 12 10 11 12 13 13 13 13 10/30 Write block 232 len 24576 data: 18 16 14 14 19 21 11/30 Read block 358 len 32768 12/30 Write block 236 len 36864 data: 20 22 18 19 20 22 18 19 19 13/30 Write block 118 len 4096 data: 14 14/30 Flush 15/30 Write block 447 len 4096 data: 27 16/30 Read block 461 len 32768 17/30 Write block 62 len 16384 data: 23 22 19 19 18/30 Write block 331 len 40960 data: 15 15 19 17 19 22 20 22 20 18 19/30 Flush 20/30 Write block 267 len 4096 data: 16 21/30 Read block 10 len 16384 22/30 Read block 50 len 20480 23/30 Flush 24/30 Write block 18 len 40960 data: 14 18 15 14 12 12 13 13 13 10 25/30 Read block 452 len 28672 26/30 Read block 64 len 8192 27/30 Read block 498 len 20480 28/30 Write block 391 len 40960 data: 8 7 6 10 8 10 10 10 9 11 29/30 Write block 102 len 8192 data: 12 16 30/30 Read block 467 len 24576 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 6 5 7 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:47.854482645Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1936} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 336 336 336 T T T 1 020-039 182 182 182 333 333 333 T T T 2 040-059 177 177 177 324 324 324 F F F 3 060-079 186 186 186 337 337 337 F F F 4 080-099 182 182 182 333 333 333 F F F 5 100-119 186 186 186 336 336 336 T T T 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 182 182 182 333 333 333 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 186 186 186 336 336 336 F F F 11 220-239 186 186 186 336 336 336 F F F 12 240-259 186 186 186 336 336 336 F F F 13 260-279 186 186 186 338 338 338 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 184 184 184 334 334 334 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 180 180 180 328 328 328 F F F 19 380-399 182 182 182 333 333 333 T T T 20 400-419 184 184 184 334 334 334 T T T 21 420-439 184 184 184 334 334 334 F F F 22 440-459 186 186 186 337 337 337 F F F 23 460-479 185 185 185 335 335 335 F F F 24 480-499 186 186 186 336 336 336 F F F 25 500-519 182 182 182 333 333 333 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 186 186 186 336 336 336 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 185 185 185 335 335 335 F F F Max gen: 186, Max flush: 338 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 187 187 187 339 339 339 F F F 2 040-059 177 177 177 324 324 324 F F F 3 060-079 186 186 186 337 337 337 F F F 4 080-099 182 182 182 333 333 333 F F F 5 100-119 187 187 187 339 339 339 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 179 179 179 327 327 327 F F F 8 160-179 182 182 182 333 333 333 F F F 9 180-199 177 177 177 324 324 324 F F F 10 200-219 186 186 186 336 336 336 F F F 11 220-239 186 186 186 336 336 336 F F F 12 240-259 186 186 186 336 336 336 F F F 13 260-279 186 186 186 338 338 338 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 184 184 184 334 334 334 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 180 180 180 328 328 328 F F F 19 380-399 187 187 187 339 339 339 F F F 20 400-419 187 187 187 339 339 339 F F F 21 420-439 184 184 184 334 334 334 F F F 22 440-459 186 186 186 337 337 337 F F F 23 460-479 185 185 185 335 335 335 F F F 24 480-499 186 186 186 336 336 336 F F F 25 500-519 182 182 182 333 333 333 F F F 26 520-539 175 175 175 320 320 320 F F F 27 540-559 186 186 186 336 336 336 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 185 185 185 335 335 335 F F F Max gen: 187, Max flush: 339 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.137359191Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.18294984Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.183653553Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.183690138Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.183705566Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.183720364Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} {"msg":"Crucible 1773a2cd-e16e-4651-9a2f-51c50ca65fa3 has session id: 1cc541ff-c0cf-4ea8-80ef-059db9211ee6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.184337869Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Upstairs opts: Upstairs UUID: 1773a2cd-e16e-4651-9a2f-51c50ca65fa3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.184390382Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Crucible stats registered with UUID: 1773a2cd-e16e-4651-9a2f-51c50ca65fa3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.184420588Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:52.184447664Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.1848381Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.184870406Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} {"msg":"1773a2cd-e16e-4651-9a2f-51c50ca65fa3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.184890283Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.18491123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"0","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.184955844Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"0","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.18498411Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"1","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185015206Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"1","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185040223Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"2","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185067789Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"2","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185094795Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185124491Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185155347Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44151)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185189892Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61542)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.185453776Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"ds_connection connected from Ok(127.0.0.1:38066)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.18549986Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.186024067Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"2","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.186168377Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"1","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.186203093Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"0","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.186666249Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"2","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.186695395Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"0","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.186819358Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"1","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187283084Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187308171Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[0]R flush_numbers[0..12]: [339, 339, 324, 337, 333, 339, 327, 327, 333, 324, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187322719Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[0]R generation[0..12]: [187, 187, 177, 186, 182, 187, 179, 179, 182, 177, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187338687Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187353574Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[1]R flush_numbers[0..12]: [339, 339, 324, 337, 333, 339, 327, 327, 333, 324, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187371282Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[1]R generation[0..12]: [187, 187, 177, 186, 182, 187, 179, 179, 182, 177, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.18738585Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187400448Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[2]R flush_numbers[0..12]: [339, 339, 324, 337, 333, 339, 327, 327, 333, 324, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187414766Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[2]R generation[0..12]: [187, 187, 177, 186, 182, 187, 179, 179, 182, 177, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187428964Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187443362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.18745675Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187470418Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187483707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187497205Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"":"downstairs","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187510843Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Transition from WaitQuorum to Active",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.18753073Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"0","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187544938Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"1","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187558856Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"client":"2","":"downstairs","session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187572654Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187587262Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"1773a2cd-e16e-4651-9a2f-51c50ca65fa3 is now active with session: 1cc541ff-c0cf-4ea8-80ef-059db9211ee6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187600671Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187612929Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944,"session_id":"1cc541ff-c0cf-4ea8-80ef-059db9211ee6"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187625497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} {"msg":"Activated sub_volume 1773a2cd-e16e-4651-9a2f-51c50ca65fa3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:52.187637095Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 176 len 32768 data: 9 9 9 14 17 20 18 15 02/30 Read block 208 len 16384 03/30 Write block 34 len 32768 data: 6 5 7 6 8 11 13 18 04/30 Read block 282 len 24576 05/30 Read block 512 len 24576 06/30 Read block 144 len 8192 07/30 Write block 528 len 40960 data: 16 20 23 21 21 21 20 17 17 18 08/30 Read block 164 len 24576 09/30 Write block 82 len 24576 data: 14 15 17 13 17 13 10/30 Write block 31 len 28672 data: 6 7 8 7 6 8 7 11/30 Read block 85 len 24576 12/30 Write block 76 len 40960 data: 14 14 14 14 15 17 15 16 18 14 13/30 Read block 60 len 8192 14/30 Flush 15/30 Write block 377 len 32768 data: 6 7 11 14 15 16 14 12 16/30 Write block 466 len 12288 data: 16 12 13 17/30 Write block 557 len 12288 data: 11 11 13 18/30 Write block 302 len 40960 data: 12 12 13 12 9 9 10 8 9 11 19/30 Write block 179 len 28672 data: 15 18 21 19 16 13 12 20/30 Flush 21/30 Write block 502 len 16384 data: 15 20 17 14 22/30 Write block 457 len 12288 data: 15 16 15 23/30 Write block 372 len 36864 data: 13 11 7 6 8 7 8 12 15 24/30 Write block 158 len 20480 data: 19 17 17 17 15 25/30 Write block 441 len 4096 data: 15 26/30 Flush 27/30 Write block 77 len 36864 data: 15 15 15 16 18 16 17 19 15 28/30 Read block 342 len 24576 29/30 Write block 90 len 8192 data: 16 17 30/30 Read block 107 len 4096 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:52.209139458Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1944} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 188 188 188 340 340 340 F F F 2 040-059 188 188 188 340 340 340 F F F 3 060-079 188 188 188 340 340 340 T T T 4 080-099 188 188 188 340 340 340 T T T 5 100-119 187 187 187 339 339 339 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 188 188 188 342 342 342 F F F 8 160-179 188 188 188 342 342 342 F F F 9 180-199 188 188 188 341 341 341 F F F 10 200-219 186 186 186 336 336 336 F F F 11 220-239 186 186 186 336 336 336 F F F 12 240-259 186 186 186 336 336 336 F F F 13 260-279 186 186 186 338 338 338 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 188 188 188 341 341 341 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 188 188 188 342 342 342 F F F 19 380-399 188 188 188 342 342 342 F F F 20 400-419 187 187 187 339 339 339 F F F 21 420-439 184 184 184 334 334 334 F F F 22 440-459 188 188 188 342 342 342 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 186 186 186 336 336 336 F F F 25 500-519 188 188 188 342 342 342 F F F 26 520-539 188 188 188 340 340 340 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 185 185 185 335 335 335 F F F Max gen: 188, Max flush: 342 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 188 188 188 340 340 340 F F F 2 040-059 188 188 188 340 340 340 F F F 3 060-079 189 189 189 343 343 343 F F F 4 080-099 189 189 189 343 343 343 F F F 5 100-119 187 187 187 339 339 339 F F F 6 120-139 179 179 179 327 327 327 F F F 7 140-159 188 188 188 342 342 342 F F F 8 160-179 188 188 188 342 342 342 F F F 9 180-199 188 188 188 341 341 341 F F F 10 200-219 186 186 186 336 336 336 F F F 11 220-239 186 186 186 336 336 336 F F F 12 240-259 186 186 186 336 336 336 F F F 13 260-279 186 186 186 338 338 338 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 188 188 188 341 341 341 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 188 188 188 342 342 342 F F F 19 380-399 188 188 188 342 342 342 F F F 20 400-419 187 187 187 339 339 339 F F F 21 420-439 184 184 184 334 334 334 F F F 22 440-459 188 188 188 342 342 342 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 186 186 186 336 336 336 F F F 25 500-519 188 188 188 342 342 342 F F F 26 520-539 188 188 188 340 340 340 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 185 185 185 335 335 335 F F F Max gen: 189, Max flush: 343 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.439111073Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.485992344Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.486847807Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.486885302Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.486899849Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.486914447Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} {"msg":"Crucible c389f871-0fed-4c90-af89-20657e21e9a9 has session id: ef502fb1-6713-425f-bcab-343f2898dede","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.487587165Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Upstairs opts: Upstairs UUID: c389f871-0fed-4c90-af89-20657e21e9a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.487637578Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Crucible stats registered with UUID: c389f871-0fed-4c90-af89-20657e21e9a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.487658075Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:56.487672313Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488162656Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488189592Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} {"msg":"c389f871-0fed-4c90-af89-20657e21e9a9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488266322Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488284339Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"0","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488314325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"0","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488329413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"1","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.48834771Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"1","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488363258Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"2","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488384375Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"2","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488409942Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488434289Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488458035Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"ds_connection connected from Ok(127.0.0.1:48224)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488483092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"ds_connection connected from Ok(127.0.0.1:58335)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488508658Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"ds_connection connected from Ok(127.0.0.1:35372)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.488695842Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.489285421Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"2","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.489315707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"0","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.489928053Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"0","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.489968067Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"2","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.490609989Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.490643245Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.49125918Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"1","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.491819433Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"1","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[0]R flush_numbers[0..12]: [339, 340, 340, 343, 343, 339, 327, 342, 342, 341, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492353059Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[0]R generation[0..12]: [187, 188, 188, 189, 189, 187, 179, 188, 188, 188, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492385075Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492404932Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[1]R flush_numbers[0..12]: [339, 340, 340, 343, 343, 339, 327, 342, 342, 341, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.49242324Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[1]R generation[0..12]: [187, 188, 188, 189, 189, 187, 179, 188, 188, 188, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492441317Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492459355Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[2]R flush_numbers[0..12]: [339, 340, 340, 343, 343, 339, 327, 342, 342, 341, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492476882Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[2]R generation[0..12]: [187, 188, 188, 189, 189, 187, 179, 188, 188, 188, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.49249455Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:14:56.492512318Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492532915Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492549903Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492571509Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492589177Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"":"downstairs","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492605545Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492617993Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"0","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492631711Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"1","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492645179Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"client":"2","":"downstairs","session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492658837Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492671486Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"c389f871-0fed-4c90-af89-20657e21e9a9 is now active with session: ef502fb1-6713-425f-bcab-343f2898dede","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492684764Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492696922Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952,"session_id":"ef502fb1-6713-425f-bcab-343f2898dede"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492709541Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} {"msg":"Activated sub_volume c389f871-0fed-4c90-af89-20657e21e9a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:14:56.492723648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 24576 data: 11 13 16 14 12 12 02/30 Read block 473 len 32768 03/30 Read block 450 len 12288 04/30 Write block 33 len 16384 data: 9 8 7 9 05/30 Read block 9 len 12288 06/30 Write block 264 len 4096 data: 17 07/30 Write block 67 len 16384 data: 18 16 16 14 08/30 Flush 09/30 Write block 531 len 8192 data: 22 22 10/30 Read block 259 len 28672 11/30 Write block 594 len 8192 data: 19 15 12/30 Write block 489 len 24576 data: 16 17 16 13 12 14 13/30 Read block 53 len 40960 14/30 Flush 15/30 Write block 119 len 16384 data: 19 18 20 21 16/30 Read block 494 len 20480 17/30 Write block 186 len 28672 data: 13 12 12 14 15 19 17 18/30 Read block 73 len 20480 19/30 Read block 428 len 40960 20/30 Flush 21/30 Write block 252 len 16384 data: 11 11 10 14 22/30 Write block 177 len 24576 data: 10 10 16 19 22 20 23/30 Write block 240 len 16384 data: 21 23 19 20 24/30 Read block 75 len 36864 25/30 Read block 354 len 40960 26/30 Write block 249 len 12288 data: 18 19 13 27/30 Read block 298 len 40960 28/30 Read block 477 len 40960 29/30 Write block 441 len 12288 data: 16 15 17 30/30 Write block 332 len 16384 data: 16 20 18 20 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:14:56.512931849Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1952} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 190 190 190 344 344 344 F F F 2 040-059 188 188 188 340 340 340 F F F 3 060-079 190 190 190 344 344 344 F F F 4 080-099 189 189 189 343 343 343 F F F 5 100-119 190 190 190 346 346 346 F F F 6 120-139 190 190 190 346 346 346 F F F 7 140-159 188 188 188 342 342 342 F F F 8 160-179 188 188 188 342 342 342 T T T 9 180-199 190 190 190 346 346 346 T T T 10 200-219 186 186 186 336 336 336 F F F 11 220-239 186 186 186 336 336 336 F F F 12 240-259 186 186 186 336 336 336 T T T 13 260-279 190 190 190 344 344 344 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 188 188 188 341 341 341 F F F 16 320-339 186 186 186 337 337 337 T T T 17 340-359 186 186 186 337 337 337 F F F 18 360-379 188 188 188 342 342 342 F F F 19 380-399 188 188 188 342 342 342 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 184 184 184 334 334 334 F F F 22 440-459 188 188 188 342 342 342 T T T 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 188 188 188 342 342 342 F F F 26 520-539 190 190 190 345 345 345 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 190 190 190 345 345 345 F F F Max gen: 190, Max flush: 346 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 190 190 190 344 344 344 F F F 2 040-059 188 188 188 340 340 340 F F F 3 060-079 190 190 190 344 344 344 F F F 4 080-099 189 189 189 343 343 343 F F F 5 100-119 190 190 190 346 346 346 F F F 6 120-139 190 190 190 346 346 346 F F F 7 140-159 188 188 188 342 342 342 F F F 8 160-179 191 191 191 347 347 347 F F F 9 180-199 191 191 191 347 347 347 F F F 10 200-219 186 186 186 336 336 336 F F F 11 220-239 186 186 186 336 336 336 F F F 12 240-259 191 191 191 347 347 347 F F F 13 260-279 190 190 190 344 344 344 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 188 188 188 341 341 341 F F F 16 320-339 191 191 191 347 347 347 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 188 188 188 342 342 342 F F F 19 380-399 188 188 188 342 342 342 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 184 184 184 334 334 334 F F F 22 440-459 191 191 191 347 347 347 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 188 188 188 342 342 342 F F F 26 520-539 190 190 190 345 345 345 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 190 190 190 345 345 345 F F F Max gen: 191, Max flush: 347 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.763697779Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.808134327Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.808858868Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.808893893Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.808908151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.808923009Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} {"msg":"Crucible 7489284f-c5e4-4422-bedd-f7c4c7ff7947 has session id: 48b30714-04a9-4534-aadf-d6b3d7114559","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.809440138Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Upstairs opts: Upstairs UUID: 7489284f-c5e4-4422-bedd-f7c4c7ff7947, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.809479922Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Crucible stats registered with UUID: 7489284f-c5e4-4422-bedd-f7c4c7ff7947","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.80949713Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:10.809516337Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.809906194Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.80993171Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} {"msg":"7489284f-c5e4-4422-bedd-f7c4c7ff7947 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.81000865Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810029017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"0","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810058913Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"0","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810073461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"1","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810088239Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"1","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810102087Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810116615Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810130203Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810144821Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810159319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:10.810509201Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:10.810544056Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"ds_connection connected from Ok(127.0.0.1:53830)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810564253Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"ds_connection connected from Ok(127.0.0.1:42437)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810580621Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:10.810597219Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810609667Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.810981876Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"0","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.811009362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"1","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.811710175Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"0","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.81175015Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"1","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.81240643Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:10.812440185Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.81118365Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"ds_connection connected from Ok(127.0.0.1:61874)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.811270958Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.812438508Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.812965215Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[0]R flush_numbers[0..12]: [339, 344, 340, 344, 343, 346, 346, 342, 347, 347, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813558444Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[0]R generation[0..12]: [187, 190, 188, 190, 189, 190, 190, 188, 191, 191, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813591299Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813610487Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[1]R flush_numbers[0..12]: [339, 344, 340, 344, 343, 346, 346, 342, 347, 347, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813628584Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[1]R generation[0..12]: [187, 190, 188, 190, 189, 190, 190, 188, 191, 191, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813646902Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813664849Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[2]R flush_numbers[0..12]: [339, 344, 340, 344, 343, 346, 346, 342, 347, 347, 336, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813682606Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[2]R generation[0..12]: [187, 190, 188, 190, 189, 190, 190, 188, 191, 191, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813700544Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813719092Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813735689Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813752727Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813769035Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813791982Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"":"downstairs","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813808669Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813824017Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"0","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813843415Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"1","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813860532Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"client":"2","":"downstairs","session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813878859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813894467Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"7489284f-c5e4-4422-bedd-f7c4c7ff7947 is now active with session: 48b30714-04a9-4534-aadf-d6b3d7114559","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813910815Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813926053Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960,"session_id":"48b30714-04a9-4534-aadf-d6b3d7114559"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813941591Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} {"msg":"Activated sub_volume 7489284f-c5e4-4422-bedd-f7c4c7ff7947","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:20.813955859Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 32768 data: 13 15 15 18 16 15 13 17 02/30 Read block 6 len 28672 03/30 Write block 251 len 32768 data: 14 12 12 11 15 15 17 19 04/30 Flush 05/30 Write block 384 len 28672 data: 13 13 13 12 13 7 9 06/30 Read block 400 len 8192 07/30 Flush 08/30 Write block 47 len 40960 data: 20 19 14 16 15 14 16 16 19 17 09/30 Read block 200 len 36864 10/30 Read block 534 len 4096 11/30 Write block 366 len 32768 data: 11 14 13 15 15 14 14 12 12/30 Write block 527 len 4096 data: 14 13/30 Read block 178 len 20480 14/30 Read block 164 len 40960 15/30 Read block 203 len 8192 16/30 Write block 381 len 16384 data: 16 17 15 14 17/30 Write block 131 len 32768 data: 12 12 16 15 17 13 13 12 18/30 Write block 331 len 4096 data: 16 19/30 Read block 509 len 16384 20/30 Flush 21/30 Flush 22/30 Write block 201 len 12288 data: 16 15 14 23/30 Read block 491 len 12288 24/30 Write block 117 len 40960 data: 14 15 20 19 21 22 18 19 22 19 25/30 Read block 214 len 32768 26/30 Read block 508 len 12288 27/30 Write block 31 len 24576 data: 7 8 10 9 8 10 28/30 Write block 436 len 40960 data: 10 9 8 8 12 17 16 18 21 22 29/30 Write block 312 len 28672 data: 11 10 11 9 10 8 10 30/30 Read block 2 len 36864 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 9 7 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:20.83547168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1960} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 190 190 190 344 344 344 T T T 2 040-059 192 192 192 350 350 350 F F F 3 060-079 190 190 190 344 344 344 F F F 4 080-099 189 189 189 343 343 343 F F F 5 100-119 190 190 190 346 346 346 T T T 6 120-139 192 192 192 350 350 350 T T T 7 140-159 188 188 188 342 342 342 F F F 8 160-179 191 191 191 347 347 347 F F F 9 180-199 191 191 191 347 347 347 F F F 10 200-219 186 186 186 336 336 336 T T T 11 220-239 186 186 186 336 336 336 F F F 12 240-259 192 192 192 348 348 348 F F F 13 260-279 190 190 190 344 344 344 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 188 188 188 341 341 341 T T T 16 320-339 192 192 192 350 350 350 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 192 192 192 350 350 350 F F F 19 380-399 192 192 192 350 350 350 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 184 184 184 334 334 334 T T T 22 440-459 191 191 191 347 347 347 T T T 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 188 188 188 342 342 342 F F F 26 520-539 192 192 192 350 350 350 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 190 190 190 345 345 345 F F F Max gen: 192, Max flush: 350 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 193 193 193 351 351 351 F F F 2 040-059 192 192 192 350 350 350 F F F 3 060-079 190 190 190 344 344 344 F F F 4 080-099 189 189 189 343 343 343 F F F 5 100-119 193 193 193 351 351 351 F F F 6 120-139 193 193 193 351 351 351 F F F 7 140-159 188 188 188 342 342 342 F F F 8 160-179 191 191 191 347 347 347 F F F 9 180-199 191 191 191 347 347 347 F F F 10 200-219 193 193 193 351 351 351 F F F 11 220-239 186 186 186 336 336 336 F F F 12 240-259 192 192 192 348 348 348 F F F 13 260-279 190 190 190 344 344 344 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 193 193 193 351 351 351 F F F 16 320-339 192 192 192 350 350 350 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 192 192 192 350 350 350 F F F 19 380-399 192 192 192 350 350 350 F F F 20 400-419 190 190 190 344 344 344 F F F 21 420-439 193 193 193 351 351 351 F F F 22 440-459 193 193 193 351 351 351 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 188 188 188 342 342 342 F F F 26 520-539 192 192 192 350 350 350 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 185 185 185 335 335 335 F F F 29 580-599 190 190 190 345 345 345 F F F Max gen: 193, Max flush: 351 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.091629945Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.138327123Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.139134032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.139170097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.139184674Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.139199292Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} {"msg":"Crucible b515ac14-8493-46fd-b8a8-21dd9357cce5 has session id: f60b86e0-829f-48cb-bda4-18a3346fe0b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.139750467Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Upstairs opts: Upstairs UUID: b515ac14-8493-46fd-b8a8-21dd9357cce5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.139791581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Crucible stats registered with UUID: b515ac14-8493-46fd-b8a8-21dd9357cce5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.139818877Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:35.139841024Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140259517Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140294392Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} {"msg":"b515ac14-8493-46fd-b8a8-21dd9357cce5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140323408Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140353784Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"0","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140409996Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"0","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140445581Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"1","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140481316Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"1","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140554336Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"2","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140583172Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"2","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140646173Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140776906Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.14081727Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44581)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140846976Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54689)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.140871762Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48708)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.14088883Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.141216095Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"2","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.141242931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"0","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.141896362Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"2","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.141924658Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"0","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.142503878Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.142594896Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.143310897Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"1","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.143817707Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"1","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[0]R flush_numbers[0..12]: [339, 351, 350, 344, 343, 351, 351, 342, 347, 347, 351, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.1443097Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[0]R generation[0..12]: [187, 193, 192, 190, 189, 193, 193, 188, 191, 191, 193, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144331057Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144346665Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[1]R flush_numbers[0..12]: [339, 351, 350, 344, 343, 351, 351, 342, 347, 347, 351, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144361522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[1]R generation[0..12]: [187, 193, 192, 190, 189, 193, 193, 188, 191, 191, 193, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144375951Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144390269Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[2]R flush_numbers[0..12]: [339, 351, 350, 344, 343, 351, 351, 342, 347, 347, 351, 336]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144404607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[2]R generation[0..12]: [187, 193, 192, 190, 189, 193, 193, 188, 191, 191, 193, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144419045Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144437662Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.14445183Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144465638Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0Wait for a query_work_queue command to finish before sending IO "} {"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144487805Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144501083Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"":"downstairs","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144515032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.14452736Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"0","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144541168Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"1","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144554826Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"client":"2","":"downstairs","session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144568524Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144581032Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"b515ac14-8493-46fd-b8a8-21dd9357cce5 is now active with session: f60b86e0-829f-48cb-bda4-18a3346fe0b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144594081Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144606519Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968,"session_id":"f60b86e0-829f-48cb-bda4-18a3346fe0b0"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144618987Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} {"msg":"Activated sub_volume b515ac14-8493-46fd-b8a8-21dd9357cce5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:35.144630425Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 149 len 8192 data: 15 15 02/30 Write block 110 len 12288 data: 14 13 10 03/30 Write block 264 len 32768 data: 18 18 17 17 17 14 16 16 04/30 Read block 140 len 28672 05/30 Write block 311 len 40960 data: 12 12 11 12 10 11 9 11 10 10 06/30 Flush 07/30 Write block 370 len 4096 data: 16 08/30 Write block 217 len 40960 data: 14 14 11 13 14 15 15 16 16 16 09/30 Read block 158 len 4096 10/30 Write block 386 len 20480 data: 14 13 14 8 10 11/30 Read block 376 len 24576 12/30 Read block 71 len 20480 13/30 Read block 318 len 24576 14/30 Read block 358 len 12288 15/30 Write block 65 len 16384 data: 20 17 19 17 16/30 Read block 548 len 12288 17/30 Write block 388 len 16384 data: 15 9 11 9 18/30 Write block 576 len 40960 data: 10 11 14 15 12 10 10 10 11 14 19/30 Write block 457 len 8192 data: 16 17 20/30 Read block 17 len 40960 21/30 Write block 516 len 36864 data: 19 16 16 12 16 15 13 9 10 22/30 Flush 23/30 Write block 206 len 36864 data: 14 14 15 17 15 11 14 17 17 24/30 Read block 79 len 32768 25/30 Read block 259 len 16384 26/30 Write block 417 len 36864 data: 12 14 13 11 13 15 14 14 16 27/30 Read block 282 len 4096 28/30 Read block 127 len 36864 29/30 Write block 187 len 40960 data: 13 13 15 16 20 18 19 16 14 17 30/30 Write block 34 len 28672 data: 10 9 11 8 9 12 14 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:35.166600664Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1968} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 193 193 193 351 351 351 T T T 2 040-059 192 192 192 350 350 350 T T T 3 060-079 194 194 194 353 353 353 F F F 4 080-099 189 189 189 343 343 343 F F F 5 100-119 194 194 194 352 352 352 F F F 6 120-139 193 193 193 351 351 351 F F F 7 140-159 194 194 194 352 352 352 F F F 8 160-179 191 191 191 347 347 347 F F F 9 180-199 191 191 191 347 347 347 T T T 10 200-219 194 194 194 353 353 353 T T T 11 220-239 194 194 194 353 353 353 F F F 12 240-259 192 192 192 348 348 348 F F F 13 260-279 194 194 194 352 352 352 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 194 194 194 352 352 352 F F F 16 320-339 194 194 194 352 352 352 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 194 194 194 353 353 353 F F F 19 380-399 194 194 194 353 353 353 F F F 20 400-419 190 190 190 344 344 344 T T T 21 420-439 193 193 193 351 351 351 T T T 22 440-459 194 194 194 353 353 353 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 194 194 194 353 353 353 F F F 26 520-539 194 194 194 353 353 353 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 194 194 194 353 353 353 F F F 29 580-599 194 194 194 353 353 353 F F F Max gen: 194, Max flush: 353 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 195 195 195 354 354 354 F F F 2 040-059 195 195 195 354 354 354 F F F 3 060-079 194 194 194 353 353 353 F F F 4 080-099 189 189 189 343 343 343 F F F 5 100-119 194 194 194 352 352 352 F F F 6 120-139 193 193 193 351 351 351 F F F 7 140-159 194 194 194 352 352 352 F F F 8 160-179 191 191 191 347 347 347 F F F 9 180-199 195 195 195 354 354 354 F F F 10 200-219 195 195 195 354 354 354 F F F 11 220-239 194 194 194 353 353 353 F F F 12 240-259 192 192 192 348 348 348 F F F 13 260-279 194 194 194 352 352 352 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 194 194 194 352 352 352 F F F 16 320-339 194 194 194 352 352 352 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 194 194 194 353 353 353 F F F 19 380-399 194 194 194 353 353 353 F F F 20 400-419 195 195 195 354 354 354 F F F 21 420-439 195 195 195 354 354 354 F F F 22 440-459 194 194 194 353 353 353 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 194 194 194 353 353 353 F F F 26 520-539 194 194 194 353 353 353 F F F 27 540-559 188 188 188 341 341 341 F F F 28 560-579 194 194 194 353 353 353 F F F 29 580-599 194 194 194 353 353 353 F F F Max gen: 195, Max flush: 354 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.422375718Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:39.465455983Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.465853238Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.497522902Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.498287597Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.498332831Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.498357078Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.498375775Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} {"msg":"Crucible 088d73fe-7370-408a-9bb4-2dadc045af0f has session id: b376e7cd-42a2-474c-ae3c-1196c0ffe8ba","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.498952936Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Upstairs opts: Upstairs UUID: 088d73fe-7370-408a-9bb4-2dadc045af0f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499001149Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Crucible stats registered with UUID: 088d73fe-7370-408a-9bb4-2dadc045af0f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499028706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:39.499050363Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499467845Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499495321Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} {"msg":"088d73fe-7370-408a-9bb4-2dadc045af0f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499556923Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.49957605Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"0","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499606846Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"0","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499622134Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"1","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499645151Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"1","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499670957Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"2","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499695164Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"2","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.4997207Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499745167Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499768244Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:64693)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.49979328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:38751)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.499928522Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:46238)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.500037307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.500511471Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"1","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.500550056Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"2","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.500697706Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"0","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.501297274Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"1","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.50132607Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"2","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.501342217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"0","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.501897731Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.501923827Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[0]R flush_numbers[0..12]: [339, 354, 354, 353, 343, 352, 351, 352, 347, 354, 354, 353]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.501990288Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[0]R generation[0..12]: [187, 195, 195, 194, 189, 194, 193, 194, 191, 195, 195, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502011705Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502027163Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[1]R flush_numbers[0..12]: [339, 354, 354, 353, 343, 352, 351, 352, 347, 354, 354, 353]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502041641Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[1]R generation[0..12]: [187, 195, 195, 194, 189, 194, 193, 194, 191, 195, 195, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502055759Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502070217Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[2]R flush_numbers[0..12]: [339, 354, 354, 353, 343, 352, 351, 352, 347, 354, 354, 353]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502084325Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[2]R generation[0..12]: [187, 195, 195, 194, 189, 194, 193, 194, 191, 195, 195, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502098323Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502112531Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502126729Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-18T05:15:39.502147726Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Next flush: 355","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502161424Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502175043Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"":"downstairs","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502188931Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502204449Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"0","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502218497Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"1","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502232335Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"client":"2","":"downstairs","session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502245963Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502258661Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"088d73fe-7370-408a-9bb4-2dadc045af0f is now active with session: b376e7cd-42a2-474c-ae3c-1196c0ffe8ba","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502271969Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502284568Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976,"session_id":"b376e7cd-42a2-474c-ae3c-1196c0ffe8ba"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502301125Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} {"msg":"Activated sub_volume 088d73fe-7370-408a-9bb4-2dadc045af0f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:39.502313124Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 218 len 40960 data: 15 12 14 15 16 16 17 17 17 17 02/30 Write block 192 len 16384 data: 19 20 17 15 03/30 Write block 571 len 8192 data: 15 14 04/30 Flush 05/30 Write block 327 len 28672 data: 10 10 13 15 17 17 21 06/30 Read block 139 len 20480 07/30 Write block 539 len 4096 data: 19 08/30 Read block 470 len 16384 09/30 Write block 400 len 4096 data: 12 10/30 Read block 52 len 4096 11/30 Write block 178 len 40960 data: 11 17 20 23 21 17 14 13 14 14 12/30 Read block 350 len 28672 13/30 Read block 151 len 28672 14/30 Read block 203 len 24576 15/30 Read block 405 len 16384 16/30 Read block 84 len 28672 17/30 Flush 18/30 Write block 553 len 40960 data: 11 11 12 11 12 12 14 14 16 18 19/30 Write block 81 len 4096 data: 19 20/30 Read block 346 len 20480 21/30 Write block 518 len 12288 data: 17 13 17 22/30 Read block 386 len 12288 23/30 Read block 407 len 4096 24/30 Read block 365 len 32768 25/30 Write block 192 len 36864 data: 20 21 18 16 18 14 17 16 14 26/30 Write block 536 len 24576 data: 18 19 17 20 15 15 27/30 Read block 287 len 8192 28/30 Read block 522 len 36864 29/30 Read block 71 len 32768 30/30 Write block 385 len 20480 data: 14 15 14 16 10 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:39.523105944Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1976} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 195 195 195 354 354 354 F F F 2 040-059 195 195 195 354 354 354 F F F 3 060-079 194 194 194 353 353 353 F F F 4 080-099 189 189 189 343 343 343 T T T 5 100-119 194 194 194 352 352 352 F F F 6 120-139 193 193 193 351 351 351 F F F 7 140-159 194 194 194 352 352 352 F F F 8 160-179 196 196 196 356 356 356 F F F 9 180-199 196 196 196 356 356 356 T T T 10 200-219 196 196 196 355 355 355 T T T 11 220-239 196 196 196 355 355 355 F F F 12 240-259 192 192 192 348 348 348 F F F 13 260-279 194 194 194 352 352 352 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 194 194 194 352 352 352 F F F 16 320-339 196 196 196 356 356 356 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 194 194 194 353 353 353 F F F 19 380-399 194 194 194 353 353 353 T T T 20 400-419 196 196 196 356 356 356 F F F 21 420-439 195 195 195 354 354 354 F F F 22 440-459 194 194 194 353 353 353 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 194 194 194 353 353 353 T T T 26 520-539 196 196 196 356 356 356 T T T 27 540-559 188 188 188 341 341 341 T T T 28 560-579 196 196 196 355 355 355 T T T 29 580-599 194 194 194 353 353 353 F F F Max gen: 196, Max flush: 356 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 F F F 1 020-039 195 195 195 354 354 354 F F F 2 040-059 195 195 195 354 354 354 F F F 3 060-079 194 194 194 353 353 353 F F F 4 080-099 197 197 197 357 357 357 F F F 5 100-119 194 194 194 352 352 352 F F F 6 120-139 193 193 193 351 351 351 F F F 7 140-159 194 194 194 352 352 352 F F F 8 160-179 196 196 196 356 356 356 F F F 9 180-199 197 197 197 357 357 357 F F F 10 200-219 197 197 197 357 357 357 F F F 11 220-239 196 196 196 355 355 355 F F F 12 240-259 192 192 192 348 348 348 F F F 13 260-279 194 194 194 352 352 352 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 194 194 194 352 352 352 F F F 16 320-339 196 196 196 356 356 356 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 194 194 194 353 353 353 F F F 19 380-399 197 197 197 357 357 357 F F F 20 400-419 196 196 196 356 356 356 F F F 21 420-439 195 195 195 354 354 354 F F F 22 440-459 194 194 194 353 353 353 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 197 197 197 357 357 357 F F F 26 520-539 197 197 197 357 357 357 F F F 27 540-559 197 197 197 357 357 357 F F F 28 560-579 197 197 197 357 357 357 F F F 29 580-599 194 194 194 353 353 353 F F F Max gen: 197, Max flush: 357 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.81718749Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:43.860598399Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.861067365Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.893074242Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.893700816Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7f7b6249fe377e7a3a2ebeaac107efeeef66e4b7\",\n git_commit_timestamp: \"2025-03-18T04:43:03.000000000Z\",\n git_branch: \"renovate/libc-0.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.893741911Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.893759368Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.893777296Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} {"msg":"Crucible 35173f92-83d5-4bfc-b9ef-75ad28e6ad40 has session id: 04a140ed-2bea-4ffb-8581-109ec697c4fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894339698Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Upstairs opts: Upstairs UUID: 35173f92-83d5-4bfc-b9ef-75ad28e6ad40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894383772Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Crucible stats registered with UUID: 35173f92-83d5-4bfc-b9ef-75ad28e6ad40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894408309Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:43.894433556Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894842499Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894867646Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} {"msg":"35173f92-83d5-4bfc-b9ef-75ad28e6ad40 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894918589Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894943795Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"0","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894971991Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"0","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.894987319Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"1","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895024294Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"1","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895040252Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"2","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.89505478Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"2","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895068538Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895083286Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895097754Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50897)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895118111Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:55120)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895145918Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:42332)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.895323673Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.896053693Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"1","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.896095307Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"2","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.896197403Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"0","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.89672413Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"1","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.896764235Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"2","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.896973436Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"0","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897276934Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897386649Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[0]R flush_numbers[0..12]: [339, 354, 354, 353, 357, 352, 351, 352, 356, 357, 357, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897499004Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[0]R generation[0..12]: [187, 195, 195, 194, 197, 194, 193, 194, 196, 197, 197, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897520681Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897536699Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[1]R flush_numbers[0..12]: [339, 354, 354, 353, 357, 352, 351, 352, 356, 357, 357, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897551187Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[1]R generation[0..12]: [187, 195, 195, 194, 197, 194, 193, 194, 196, 197, 197, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897565925Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897580583Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[2]R flush_numbers[0..12]: [339, 354, 354, 353, 357, 352, 351, 352, 356, 357, 357, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.89759841Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[2]R generation[0..12]: [187, 195, 195, 194, 197, 194, 193, 194, 196, 197, 197, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897614048Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897628476Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897647443Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:15:43.8976728Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Next flush: 358","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897699676Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897725623Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"":"downstairs","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897752199Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897777576Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"0","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897805632Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"1","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897832648Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"client":"2","":"downstairs","session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897859354Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"timeDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"2025-03-18T05:15:43.897884461Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"35173f92-83d5-4bfc-b9ef-75ad28e6ad40 is now active with session: 04a140ed-2bea-4ffb-8581-109ec697c4fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897914227Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897937974Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984,"session_id":"04a140ed-2bea-4ffb-8581-109ec697c4fd"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.897951561Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} {"msg":"Activated sub_volume 35173f92-83d5-4bfc-b9ef-75ad28e6ad40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:15:43.89796328Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 16384 data: 15 14 19 20 02/30 Write block 23 len 32768 data: 13 14 14 14 11 7 7 5 03/30 Write block 401 len 12288 data: 12 14 17 04/30 Write block 579 len 28672 data: 16 13 11 11 11 12 15 05/30 Read block 199 len 32768 06/30 Write block 7 len 12288 data: 11 11 16 07/30 Write block 383 len 24576 data: 16 15 15 16 15 17 08/30 Read block 255 len 24576 09/30 Write block 257 len 32768 data: 18 20 23 22 17 18 16 19 10/30 Read block 255 len 20480 11/30 Write block 245 len 4096 data: 22 12/30 Read block 414 len 28672 13/30 Read block 28 len 20480 14/30 Write block 137 len 40960 data: 14 13 10 11 11 10 11 12 14 13 15/30 Write block 569 len 28672 data: 14 14 16 15 12 12 13 16/30 Write block 60 len 12288 data: 18 21 24 17/30 Write block 399 len 4096 data: 10 18/30 Read block 126 len 28672 19/30 Write block 306 len 8192 data: 10 10 20/30 Read block 190 len 40960 21/30 Read block 437 len 28672 22/30 Read block 8 len 20480 23/30 Read block 526 len 20480 24/30 Read block 506 len 8192 25/30 Read block 66 len 36864 26/30 Read block 500 len 36864 27/30 Read block 127 len 40960 28/30 Write block 131 len 40960 data: 13 13 17 16 18 14 15 14 11 12 29/30 Read block 418 len 36864 30/30 Write block 515 len 16384 data: 23 20 17 18 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 29 28 29 86 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:15:43.917918025Z","hostname":"w-01JPKSWPKZ2EFKZTKYRE72VZP5","pid":1984} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 339 339 339 T T T 1 020-039 195 195 195 354 354 354 T T T 2 040-059 195 195 195 354 354 354 F F F 3 060-079 194 194 194 353 353 353 T T T 4 080-099 197 197 197 357 357 357 F F F 5 100-119 194 194 194 352 352 352 F F F 6 120-139 193 193 193 351 351 351 T T T 7 140-159 194 194 194 352 352 352 T T T 8 160-179 196 196 196 356 356 356 F F F 9 180-199 197 197 197 357 357 357 F F F 10 200-219 197 197 197 357 357 357 F F F 11 220-239 196 196 196 355 355 355 F F F 12 240-259 192 192 192 348 348 348 T T T 13 260-279 194 194 194 352 352 352 T T T 14 280-299 184 184 184 334 334 334 F F F 15 300-319 194 194 194 352 352 352 T T T 16 320-339 196 196 196 356 356 356 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 194 194 194 353 353 353 F F F 19 380-399 197 197 197 357 357 357 T T T 20 400-419 196 196 196 356 356 356 T T T 21 420-439 195 195 195 354 354 354 F F F 22 440-459 194 194 194 353 353 353 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 197 197 197 357 357 357 T T T 26 520-539 197 197 197 357 357 357 F F F 27 540-559 197 197 197 357 357 357 F F F 28 560-579 197 197 197 357 357 357 T T T 29 580-599 194 194 194 353 353 353 T T T Max gen: 197, Max flush: 357 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 358 358 358 F F F 1 020-039 198 198 198 358 358 358 F F F 2 040-059 195 195 195 354 354 354 F F F 3 060-079 198 198 198 358 358 358 F F F 4 080-099 197 197 197 357 357 357 F F F 5 100-119 194 194 194 352 352 352 F F F 6 120-139 198 198 198 358 358 358 F F F 7 140-159 198 198 198 358 358 358 F F F 8 160-179 196 196 196 356 356 356 F F F 9 180-199 197 197 197 357 357 357 F F F 10 200-219 197 197 197 357 357 357 F F F 11 220-239 196 196 196 355 355 355 F F F 12 240-259 198 198 198 358 358 358 F F F 13 260-279 198 198 198 358 358 358 F F F 14 280-299 184 184 184 334 334 334 F F F 15 300-319 198 198 198 358 358 358 F F F 16 320-339 196 196 196 356 356 356 F F F 17 340-359 186 186 186 337 337 337 F F F 18 360-379 194 194 194 353 353 353 F F F 19 380-399 198 198 198 358 358 358 F F F 20 400-419 198 198 198 358 358 358 F F F 21 420-439 195 195 195 354 354 354 F F F 22 440-459 194 194 194 353 353 353 F F F 23 460-479 188 188 188 341 341 341 F F F 24 480-499 190 190 190 345 345 345 F F F 25 500-519 198 198 198 358 358 358 F F F 26 520-539 197 197 197 357 357 357 F F F 27 540-559 197 197 197 357 357 357 F F F 28 560-579 198 198 198 358 358 358 F F F 29 580-599 198 198 198 358 358 358 F F F Max gen: 198, Max flush: 358 15:35 Test duration Test completed real 15:35.228739404 user 50.351117602 sys 19.470641587 trap 0.062619126 tflt 0.078189098 dflt 0.144383466 kflt 0.006324268 lock 24:52:02.178991861 slp 1:16:44.943104994 lat 5.823545634 stop 0.069079015