Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.26686123 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.021880658 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.01918273 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1193 1194 1195 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.088944073Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.144530357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.145678387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.145774397Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.145793775Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.145809063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} {"msg":"Crucible 8dba5e95-27c6-4225-aa07-f17c06c44305 has session id: e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.146634067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Upstairs opts: Upstairs UUID: 8dba5e95-27c6-4225-aa07-f17c06c44305, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.146708299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Crucible stats registered with UUID: 8dba5e95-27c6-4225-aa07-f17c06c44305","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.146742486Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:28.146761074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.147735732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.147765839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} {"msg":"8dba5e95-27c6-4225-aa07-f17c06c44305 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.147842261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.147865139Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"0","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.147964778Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"0","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.147984926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"1","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148002834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"1","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148019103Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"2","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148036571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"2","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148062078Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148091315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148120752Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"ds_connection connected from Ok(127.0.0.1:61904)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148149259Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"ds_connection connected from Ok(127.0.0.1:42871)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148180156Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"ds_connection connected from Ok(127.0.0.1:63188)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.148211953Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.149625125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"0","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.149659672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"1","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.149680919Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"2","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.150839159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"0","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.150868985Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"2","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.150884734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"1","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.151851403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.151949833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152128414Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152150172Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.15216483Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152178679Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152192067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152206086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152219505Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152232733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152250671Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.15226432Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152277469Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152290047Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152302686Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"":"downstairs","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152315815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152327583Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"0","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152341372Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"1","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152354021Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"client":"2","":"downstairs","session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152366799Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152378808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"8dba5e95-27c6-4225-aa07-f17c06c44305 is now active with session: e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152391427Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2025-03-18T04:59:28.152403196Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198,"session_id":"e77ef7f1-c4f6-42a6-b8fb-26a3dcc1a242"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152419344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} {"msg":"Activated sub_volume 8dba5e95-27c6-4225-aa07-f17c06c44305","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:28.152430303Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:32.235095125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1198} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.267513114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.312762235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.31338778Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.313431255Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.313448873Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.313466502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} {"msg":"Crucible c0bede2b-0c16-4369-a3fe-53f38036525f has session id: 33a55f2e-a364-481a-847d-6edc430f6256","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314087107Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Upstairs opts: Upstairs UUID: c0bede2b-0c16-4369-a3fe-53f38036525f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314129163Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Crucible stats registered with UUID: c0bede2b-0c16-4369-a3fe-53f38036525f","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314146441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:32.314163829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314664467Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314689024Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} {"msg":"c0bede2b-0c16-4369-a3fe-53f38036525f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314746378Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314773535Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"0","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314805672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"0","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314820561Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"1","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314835019Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"1","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.3149179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"2","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314951897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"2","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.314980104Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.315006421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.315030309Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"ds_connection connected from Ok(127.0.0.1:56290)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.315057306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"ds_connection connected from Ok(127.0.0.1:39266)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.315130518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"ds_connection connected from Ok(127.0.0.1:59232)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.315168764Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.315841594Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"0","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.31587714Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"1","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.316475348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"2","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.316503245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"1","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.316643351Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"0","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317131749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317242438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317261946Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"2","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317819788Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317839286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317853794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317867643Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317880911Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.31789952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317913828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317927337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317941895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317954554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317967722Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T04:59:32.317980321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.317998399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"":"downstairs","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318011438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318023427Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"0","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318036795Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"1","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318049944Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"client":"2","":"downstairs","session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318063182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318075161Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"c0bede2b-0c16-4369-a3fe-53f38036525f is now active with session: 33a55f2e-a364-481a-847d-6edc430f6256","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.31808772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318099319Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200,"session_id":"33a55f2e-a364-481a-847d-6edc430f6256"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318113957Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} {"msg":"Activated sub_volume c0bede2b-0c16-4369-a3fe-53f38036525f","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:32.318125156Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 8192 data: 2 2 02/30 Write block 170 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 339 len 28672 04/30 Write block 43 len 40960 data: 2 2 2 2 2 2 2 2 2 2 05/30 Read block 168 len 28672 06/30 Flush 07/30 Flush 08/30 Write block 29 len 16384 data: 2 2 2 2 09/30 Read block 242 len 36864 10/30 Write block 304 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Write block 534 len 4096 data: 2 12/30 Flush 13/30 Write block 369 len 16384 data: 2 2 2 2 14/30 Read block 343 len 16384 15/30 Write block 584 len 24576 data: 2 2 2 2 2 2 16/30 Read block 465 len 32768 17/30 Write block 7 len 4096 data: 2 18/30 Write block 522 len 20480 data: 2 2 2 2 2 19/30 Write block 514 len 28672 data: 2 2 2 2 2 2 2 20/30 Read block 567 len 16384 21/30 Read block 163 len 12288 22/30 Read block 356 len 12288 23/30 Read block 457 len 36864 24/30 Read block 386 len 28672 25/30 Read block 153 len 36864 26/30 Read block 156 len 40960 27/30 Read block 86 len 12288 28/30 Read block 419 len 4096 29/30 Write block 493 len 40960 data: 2 2 2 2 2 2 2 2 2 3 30/30 Write block 360 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:32.339126376Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1200} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 4 4 4 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 2 2 2 2 2 2 T T T 26 520-539 2 2 2 4 4 4 T T T 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 T T T Max gen: 2, Max flush: 4 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 5 5 5 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 4 4 4 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 3 3 3 5 5 5 F F F 25 500-519 3 3 3 5 5 5 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 3 3 3 5 5 5 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.583545069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:36.626299991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.626701009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.658071837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.658647987Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.658680724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.658694592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.658708811Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} {"msg":"Crucible 2917810e-251d-45db-a8c4-c42297f79853 has session id: 33497f73-e42c-4f15-ad98-4229a6e1da35","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659276422Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Upstairs opts: Upstairs UUID: 2917810e-251d-45db-a8c4-c42297f79853, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659373861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Crucible stats registered with UUID: 2917810e-251d-45db-a8c4-c42297f79853","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659404178Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:36.659423336Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659829694Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659858841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} {"msg":"2917810e-251d-45db-a8c4-c42297f79853 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659923384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659941522Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"0","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659968959Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"0","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.659982678Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"1","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660000796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"1","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660025724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"2","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660040242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"2","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660053261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660076828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660103316Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"ds_connection connected from Ok(127.0.0.1:44298)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660396325Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"ds_connection connected from Ok(127.0.0.1:32935)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660435441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"ds_connection connected from Ok(127.0.0.1:35763)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660461588Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660938628Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"1","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.660979794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"2","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.661012731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"0","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.661587351Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"1","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.661628606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"2","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.661744134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"0","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662182549Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662305106Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[0]R flush_numbers[0..12]: [5, 4, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662429453Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[0]R generation[0..12]: [3, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662448651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662463419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[1]R flush_numbers[0..12]: [5, 4, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662477058Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[1]R generation[0..12]: [3, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662490487Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662504245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[2]R flush_numbers[0..12]: [5, 4, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662517394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[2]R generation[0..12]: [3, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662531442Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662544981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.66255798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662571148Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.66264663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662665888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"":"downstairs","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662679847Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662691715Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"0","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662704974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"1","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662717803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"client":"2","":"downstairs","session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662730742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.66274268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"2917810e-251d-45db-a8c4-c42297f79853 is now active with session: 33497f73-e42c-4f15-ad98-4229a6e1da35","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662755509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662767408Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208,"session_id":"33497f73-e42c-4f15-ad98-4229a6e1da35"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662779366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} {"msg":"Activated sub_volume 2917810e-251d-45db-a8c4-c42297f79853","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:36.662790275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 268 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 425 len 40960 03/30 Write block 196 len 20480 data: 2 2 2 2 2 04/30 Read block 73 len 24576 05/30 Write block 510 len 20480 data: 2 2 2 2 3 06/30 Write block 63 len 12288 data: 2 2 2 07/30 Read block 408 len 12288 08/30 Read block 369 len 28672 09/30 Flush 10/30 Write block 351 len 28672 data: 2 2 2 2 2 2 2 11/30 Write block 426 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Write block 304 len 36864 data: 3 3 3 3 3 3 3 3 2 13/30 Read block 320 len 12288 14/30 Write block 7 len 36864 data: 3 2 2 2 2 2 2 2 2 15/30 Write block 480 len 8192 data: 2 2 16/30 Read block 337 len 16384 17/30 Write block 282 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Write block 266 len 16384 data: 2 2 3 3 19/30 Write block 4 len 12288 data: 2 2 2 20/30 Read block 27 len 8192 21/30 Flush 22/30 Write block 534 len 20480 data: 3 2 2 2 2 23/30 Read block 93 len 12288 24/30 Read block 355 len 20480 25/30 Write block 496 len 8192 data: 3 3 26/30 Flush 27/30 Write block 178 len 8192 data: 2 2 28/30 Read block 162 len 28672 29/30 Read block 384 len 16384 30/30 Read block 547 len 28672 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:36.68269026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 4 4 4 6 6 6 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 2 2 2 T T T 9 180-199 4 4 4 6 6 6 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 3 3 3 5 5 5 F F F Max gen: 4, Max flush: 8 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 4 4 4 6 6 6 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 5 5 5 9 9 9 F F F 9 180-199 4 4 4 6 6 6 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 3 3 3 5 5 5 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 3 3 3 5 5 5 F F F Max gen: 5, Max flush: 9 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.928119667Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.972731405Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.973395626Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.973437181Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.973453809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.973468358Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} {"msg":"Crucible 09ea32d2-f63e-472f-81ed-1bcab3cfdc97 has session id: cacc84e8-0a66-4bfa-94f7-3a166186565c","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.974017291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Upstairs opts: Upstairs UUID: 09ea32d2-f63e-472f-81ed-1bcab3cfdc97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.974063786Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Crucible stats registered with UUID: 09ea32d2-f63e-472f-81ed-1bcab3cfdc97","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.974087123Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:40.974173784Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.974653564Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.97469214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} {"msg":"09ea32d2-f63e-472f-81ed-1bcab3cfdc97 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.974745375Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.974774422Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"0","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.974821907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"0","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975147173Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"1","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975180779Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"1","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975207026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"2","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975236703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"2","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975262991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975288788Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975313026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46715)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975334773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46150)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975354751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58470)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975374609Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.975906984Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"2","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.97594058Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"0","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.976055768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"1","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.976639617Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"2","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.976680233Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"0","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.976813459Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"1","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977223906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977321776Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[0]R flush_numbers[0..12]: [7, 4, 2, 6, 1, 1, 1, 1, 9, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977391169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[0]R generation[0..12]: [4, 2, 2, 4, 1, 1, 1, 1, 5, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977410757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977425575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[1]R flush_numbers[0..12]: [7, 4, 2, 6, 1, 1, 1, 1, 9, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977439374Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[1]R generation[0..12]: [4, 2, 2, 4, 1, 1, 1, 1, 5, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977452752Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977466501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[2]R flush_numbers[0..12]: [7, 4, 2, 6, 1, 1, 1, 1, 9, 6, 6, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.97747968Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[2]R generation[0..12]: [4, 2, 2, 4, 1, 1, 1, 1, 5, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977496408Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977510826Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977524025Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977537343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977549982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977562701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"":"downstairs","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.97757567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977587398Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"0","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977600647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"1","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977613515Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"client":"2","":"downstairs","session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T04:59:40.977626784Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977643903Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"09ea32d2-f63e-472f-81ed-1bcab3cfdc97 is now active with session: cacc84e8-0a66-4bfa-94f7-3a166186565c","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977656621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.97766832Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216,"session_id":"cacc84e8-0a66-4bfa-94f7-3a166186565c"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977680159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} {"msg":"Activated sub_volume 09ea32d2-f63e-472f-81ed-1bcab3cfdc97","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:40.977691338Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 379 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Read block 261 len 28672 04/30 Read block 507 len 24576 05/30 Write block 79 len 28672 data: 2 2 2 2 2 2 2 06/30 Write block 587 len 16384 data: 3 3 3 2 07/30 Write block 398 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Write block 259 len 28672 data: 2 2 2 2 2 2 2 09/30 Read block 6 len 16384 10/30 Read block 592 len 32768 11/30 Read block 583 len 12288 12/30 Write block 102 len 4096 data: 2 13/30 Write block 380 len 12288 data: 3 3 3 14/30 Flush 15/30 Write block 316 len 32768 data: 2 2 2 2 2 2 2 2 16/30 Read block 291 len 28672 17/30 Write block 329 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Read block 418 len 36864 19/30 Flush 20/30 Write block 165 len 28672 data: 2 2 2 2 2 3 3 21/30 Read block 168 len 4096 22/30 Read block 324 len 16384 23/30 Write block 454 len 36864 data: 2 2 2 2 2 2 2 2 2 24/30 Write block 213 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Flush 26/30 Write block 546 len 40960 data: 2 2 2 2 2 2 2 2 2 2 27/30 Write block 125 len 24576 data: 2 2 2 2 2 2 28/30 Write block 509 len 12288 data: 2 3 3 29/30 Read block 577 len 32768 30/30 Write block 499 len 24576 data: 3 3 3 4 3 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:40.998621305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 6 6 6 11 11 11 F F F 4 080-099 6 6 6 11 11 11 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 13 13 13 F F F 9 180-199 4 4 4 6 6 6 F F F 10 200-219 6 6 6 13 13 13 F F F 11 220-239 6 6 6 13 13 13 F F F 12 240-259 6 6 6 11 11 11 F F F 13 260-279 6 6 6 11 11 11 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 6 6 6 12 12 12 F F F 16 320-339 6 6 6 12 12 12 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 11 11 11 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 6 6 6 13 13 13 F F F 24 480-499 4 4 4 8 8 8 T T T 25 500-519 4 4 4 6 6 6 T T T 26 520-539 4 4 4 8 8 8 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 11 11 11 F F F Max gen: 6, Max flush: 13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 2 2 2 4 4 4 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 6 6 6 11 11 11 F F F 4 080-099 6 6 6 11 11 11 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 14 14 14 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 13 13 13 F F F 9 180-199 4 4 4 6 6 6 F F F 10 200-219 6 6 6 13 13 13 F F F 11 220-239 6 6 6 13 13 13 F F F 12 240-259 6 6 6 11 11 11 F F F 13 260-279 6 6 6 11 11 11 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 6 6 6 12 12 12 F F F 16 320-339 6 6 6 12 12 12 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 11 11 11 F F F 21 420-439 4 4 4 7 7 7 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 6 6 6 13 13 13 F F F 24 480-499 7 7 7 14 14 14 F F F 25 500-519 7 7 7 14 14 14 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 7 7 7 14 14 14 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 11 11 11 F F F Max gen: 7, Max flush: 14 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.235143491Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.279788855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.280501741Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.280537397Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.280552405Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.280567134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} {"msg":"Crucible 07000d83-8bba-46f1-9a21-ef2ffb468ed5 has session id: 85edd965-7040-4a8d-b3f6-3a741621d6a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281220876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Upstairs opts: Upstairs UUID: 07000d83-8bba-46f1-9a21-ef2ffb468ed5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281268221Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Crucible stats registered with UUID: 07000d83-8bba-46f1-9a21-ef2ffb468ed5","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281292498Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:45.281314636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281833582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281859279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} {"msg":"07000d83-8bba-46f1-9a21-ef2ffb468ed5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281924362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.28194869Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"0","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281978267Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"0","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.281992835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"1","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282014503Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"1","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.28204067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"2","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282068018Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"2","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282093705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282120462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282152039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53053)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282180616Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63565)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282205423Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54346)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282373436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282886692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"1","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282915239Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"0","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.282941886Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"2","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.283535554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"1","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.283636054Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"0","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.28367534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"2","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.28414851Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284261009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[0]R flush_numbers[0..12]: [7, 4, 2, 11, 11, 11, 14, 1, 13, 6, 13, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284291576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[0]R generation[0..12]: [4, 2, 2, 6, 6, 6, 7, 1, 6, 4, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284307394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284321952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[1]R flush_numbers[0..12]: [7, 4, 2, 11, 11, 11, 14, 1, 13, 6, 13, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284340141Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[1]R generation[0..12]: [4, 2, 2, 6, 6, 6, 7, 1, 6, 4, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284353949Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284367778Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[2]R flush_numbers[0..12]: [7, 4, 2, 11, 11, 11, 14, 1, 13, 6, 13, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284381796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[2]R generation[0..12]: [4, 2, 2, 6, 6, 6, 7, 1, 6, 4, 6, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284394965Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284408773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284421382Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284434631Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284447389Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284467577Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"":"downstairs","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284480846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284492655Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"0","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284505563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"1","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284518392Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"client":"2","":"downstairs","session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284531441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284546029Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"07000d83-8bba-46f1-9a21-ef2ffb468ed5 is now active with session: 85edd965-7040-4a8d-b3f6-3a741621d6a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284558818Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284570357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224,"session_id":"85edd965-7040-4a8d-b3f6-3a741621d6a3"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284582255Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} {"msg":"Activated sub_volume 07000d83-8bba-46f1-9a21-ef2ffb468ed5","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:45.284593154Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 275 len 40960 data: 2 2 2 2 2 2 2 3 3 3 02/30 Read block 441 len 28672 03/30 Write block 310 len 4096 data: 4 04/30 Read block 593 len 16384 05/30 Write block 565 len 4096 data: 2 06/30 Flush 07/30 Write block 265 len 28672 data: 3 3 3 4 4 3 3 08/30 Write block 20 len 4096 data: 2 09/30 Write block 136 len 16384 data: 2 2 2 2 10/30 Read block 275 len 20480 11/30 Write block 249 len 8192 data: 2 2 12/30 Read block 154 len 8192 13/30 Write block 325 len 28672 data: 2 2 2 2 3 3 3 14/30 Write block 201 len 8192 data: 2 2 15/30 Write block 163 len 36864 data: 2 2 3 3 3 3 3 4 4 16/30 Write block 259 len 12288 data: 3 3 3 17/30 Read block 83 len 36864 18/30 Read block 511 len 28672 19/30 Write block 166 len 32768 data: 4 4 4 4 5 5 3 3 20/30 Read block 245 len 12288 21/30 Read block 492 len 20480 22/30 Write block 590 len 16384 data: 3 2 2 2 23/30 Read block 440 len 32768 24/30 Write block 431 len 4096 data: 3 25/30 Flush 26/30 Write block 517 len 32768 data: 3 3 3 3 2 3 3 3 27/30 Read block 440 len 28672 28/30 Write block 191 len 24576 data: 2 2 2 2 2 3 29/30 Write block 230 len 24576 data: 2 2 2 2 2 2 30/30 Write block 33 len 40960 data: 2 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Sent Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 3 3 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:45.303660056Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 8 8 8 16 16 16 T T T 2 040-059 2 2 2 2 2 2 T T T 3 060-079 6 6 6 11 11 11 F F F 4 080-099 6 6 6 11 11 11 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 4 4 4 6 6 6 T T T 10 200-219 8 8 8 16 16 16 F F F 11 220-239 6 6 6 13 13 13 T T T 12 240-259 8 8 8 16 16 16 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 8 8 8 15 15 15 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 16 16 16 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 11 11 11 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 6 6 6 13 13 13 F F F 24 480-499 7 7 7 14 14 14 F F F 25 500-519 7 7 7 14 14 14 T T T 26 520-539 4 4 4 8 8 8 T T T 27 540-559 7 7 7 14 14 14 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 8, Max flush: 16 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 9 9 9 17 17 17 F F F 2 040-059 9 9 9 17 17 17 F F F 3 060-079 6 6 6 11 11 11 F F F 4 080-099 6 6 6 11 11 11 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 9 9 9 17 17 17 F F F 10 200-219 8 8 8 16 16 16 F F F 11 220-239 9 9 9 17 17 17 F F F 12 240-259 8 8 8 16 16 16 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 8 8 8 15 15 15 F F F 15 300-319 8 8 8 15 15 15 F F F 16 320-339 8 8 8 16 16 16 F F F 17 340-359 4 4 4 7 7 7 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 6 6 6 11 11 11 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 6 6 6 13 13 13 F F F 24 480-499 7 7 7 14 14 14 F F F 25 500-519 9 9 9 17 17 17 F F F 26 520-539 9 9 9 17 17 17 F F F 27 540-559 7 7 7 14 14 14 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 9, Max flush: 17 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.546253349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:49.588798572Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.589208999Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.62007592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.620655499Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.620690966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.620705204Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.620719583Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} {"msg":"Crucible eec82687-56f4-4b66-aeaf-c540bc8cd420 has session id: 2eaf5306-efc0-481e-9497-e38da8127999","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621271005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Upstairs opts: Upstairs UUID: eec82687-56f4-4b66-aeaf-c540bc8cd420, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.6213244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Crucible stats registered with UUID: eec82687-56f4-4b66-aeaf-c540bc8cd420","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621352457Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:49.621375404Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621754355Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621787581Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} {"msg":"eec82687-56f4-4b66-aeaf-c540bc8cd420 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621843156Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621871203Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"0","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621902169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"0","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621922098Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"1","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621937886Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"1","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621956894Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"2","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.621982291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"2","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622006489Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622029766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622054574Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"ds_connection connected from Ok(127.0.0.1:42740)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622080901Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"ds_connection connected from Ok(127.0.0.1:58984)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622116827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"ds_connection connected from Ok(127.0.0.1:65449)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622265832Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622940231Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"1","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.622979707Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"2","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.623031702Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"0","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.623654237Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"1","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.623693463Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"0","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.62372445Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"2","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624243895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624275622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[0]R flush_numbers[0..12]: [7, 17, 17, 11, 11, 11, 16, 1, 16, 17, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624337546Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[0]R generation[0..12]: [4, 9, 9, 6, 6, 6, 8, 1, 8, 9, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624357333Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624371832Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[1]R flush_numbers[0..12]: [7, 17, 17, 11, 11, 11, 16, 1, 16, 17, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.62438549Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[1]R generation[0..12]: [4, 9, 9, 6, 6, 6, 8, 1, 8, 9, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624399279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624413448Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[2]R flush_numbers[0..12]: [7, 17, 17, 11, 11, 11, 16, 1, 16, 17, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624427016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[2]R generation[0..12]: [4, 9, 9, 6, 6, 6, 8, 1, 8, 9, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624440055Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624453634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624466452Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624479371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624495149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624508158Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"":"downstairs","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T04:59:49.624521456Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624540704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"0","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624554153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"1","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624567262Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"client":"2","":"downstairs","session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.6245804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624592519Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"eec82687-56f4-4b66-aeaf-c540bc8cd420 is now active with session: 2eaf5306-efc0-481e-9497-e38da8127999","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624605198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624616937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232,"session_id":"2eaf5306-efc0-481e-9497-e38da8127999"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624629045Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} {"msg":"Activated sub_volume eec82687-56f4-4b66-aeaf-c540bc8cd420","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:49.624639994Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 12288 data: 2 2 2 02/30 Read block 327 len 8192 03/30 Write block 16 len 20480 data: 2 2 2 2 3 04/30 Write block 253 len 20480 data: 2 2 2 2 2 05/30 Read block 47 len 32768 06/30 Read block 258 len 4096 07/30 Write block 81 len 12288 data: 3 3 3 08/30 Flush 09/30 Write block 154 len 28672 data: 2 2 2 2 2 2 2 10/30 Write block 110 len 12288 data: 2 2 2 11/30 Write block 410 len 40960 data: 2 2 2 2 2 2 2 2 2 2 12/30 Write block 327 len 24576 data: 3 3 4 4 4 3 13/30 Read block 161 len 16384 14/30 Write block 199 len 40960 data: 3 3 3 3 2 2 2 2 2 2 15/30 Read block 425 len 12288 16/30 Read block 575 len 4096 17/30 Read block 107 len 36864 18/30 Read block 560 len 36864 19/30 Read block 437 len 32768 20/30 Read block 295 len 20480 21/30 Write block 298 len 32768 data: 2 2 2 2 2 2 4 4 22/30 Write block 337 len 8192 data: 2 2 23/30 Flush 24/30 Write block 147 len 32768 data: 2 2 2 2 2 2 2 3 25/30 Write block 342 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Read block 535 len 8192 27/30 Write block 576 len 12288 data: 2 3 3 28/30 Read block 82 len 20480 29/30 Write block 545 len 4096 data: 2 30/30 Read block 47 len 20480 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:49.644303853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 18 18 18 F F F 1 020-039 10 10 10 18 18 18 F F F 2 040-059 9 9 9 17 17 17 F F F 3 060-079 6 6 6 11 11 11 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 19 19 19 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 10 10 10 19 19 19 T T T 8 160-179 10 10 10 19 19 19 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 10 10 10 19 19 19 F F F 11 220-239 9 9 9 17 17 17 F F F 12 240-259 10 10 10 18 18 18 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 10 10 10 19 19 19 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 10 10 10 19 19 19 F F F 17 340-359 4 4 4 7 7 7 T T T 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 10 10 10 19 19 19 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 6 6 6 13 13 13 F F F 24 480-499 7 7 7 14 14 14 F F F 25 500-519 9 9 9 17 17 17 F F F 26 520-539 9 9 9 17 17 17 F F F 27 540-559 7 7 7 14 14 14 T T T 28 560-579 10 10 10 18 18 18 T T T 29 580-599 8 8 8 16 16 16 F F F Max gen: 10, Max flush: 19 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 18 18 18 F F F 1 020-039 10 10 10 18 18 18 F F F 2 040-059 9 9 9 17 17 17 F F F 3 060-079 6 6 6 11 11 11 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 19 19 19 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 11 11 11 20 20 20 F F F 8 160-179 10 10 10 19 19 19 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 10 10 10 19 19 19 F F F 11 220-239 9 9 9 17 17 17 F F F 12 240-259 10 10 10 18 18 18 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 10 10 10 19 19 19 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 10 10 10 19 19 19 F F F 17 340-359 11 11 11 20 20 20 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 10 10 10 19 19 19 F F F 21 420-439 8 8 8 16 16 16 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 6 6 6 13 13 13 F F F 24 480-499 7 7 7 14 14 14 F F F 25 500-519 9 9 9 17 17 17 F F F 26 520-539 9 9 9 17 17 17 F F F 27 540-559 11 11 11 20 20 20 F F F 28 560-579 11 11 11 20 20 20 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 11, Max flush: 20 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.909744135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.954224856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.954901846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.954937022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.954951631Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.954966009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} {"msg":"Crucible 32e1ddd2-81eb-49ab-914c-c0271063316b has session id: 54b65455-aea5-44d2-9fa2-7730df557e90","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.955614871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Upstairs opts: Upstairs UUID: 32e1ddd2-81eb-49ab-914c-c0271063316b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.955650418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Crucible stats registered with UUID: 32e1ddd2-81eb-49ab-914c-c0271063316b","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.955666856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:53.955683234Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956147706Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956182512Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} {"msg":"32e1ddd2-81eb-49ab-914c-c0271063316b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956243986Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956261744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"0","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956289811Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"0","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956304879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"1","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956319968Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"1","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956332956Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"2","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956351974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"2","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956375942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956401539Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956425117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"ds_connection connected from Ok(127.0.0.1:61874)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956499189Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"ds_connection connected from Ok(127.0.0.1:60478)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956544864Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"ds_connection connected from Ok(127.0.0.1:39652)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.956828765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.95725609Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"0","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.957298436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"1","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.957529982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"2","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958047058Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"1","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958083914Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"0","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958127989Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"2","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958644735Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958757114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[0]R flush_numbers[0..12]: [18, 18, 17, 11, 18, 19, 16, 20, 19, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958810368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[0]R generation[0..12]: [10, 10, 9, 6, 10, 10, 8, 11, 10, 10, 10, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958829576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958844354Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[1]R flush_numbers[0..12]: [18, 18, 17, 11, 18, 19, 16, 20, 19, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958858303Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[1]R generation[0..12]: [10, 10, 9, 6, 10, 10, 8, 11, 10, 10, 10, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958871712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.9588854Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[2]R flush_numbers[0..12]: [18, 18, 17, 11, 18, 19, 16, 20, 19, 19, 19, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958902168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[2]R generation[0..12]: [10, 10, 9, 6, 10, 10, 8, 11, 10, 10, 10, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958915757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958929436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958942064Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958954983Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.958968102Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.9589809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"":"downstairs","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959010737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959025826Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"0","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959039384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"1","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959060312Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"client":"2","":"downstairs","session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959073571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959085699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"32e1ddd2-81eb-49ab-914c-c0271063316b is now active with session: 54b65455-aea5-44d2-9fa2-7730df557e90","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959098278Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959109997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240,"session_id":"54b65455-aea5-44d2-9fa2-7730df557e90"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959124365Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} {"msg":"Activated sub_volume 32e1ddd2-81eb-49ab-914c-c0271063316b","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:53.959135674Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 21 len 20480 data: 2 2 2 2 2 02/30 Write block 486 len 20480 data: 2 2 2 2 2 03/30 Write block 56 len 24576 data: 2 2 2 2 2 2 04/30 Write block 474 len 24576 data: 2 2 2 2 2 2 05/30 Write block 249 len 32768 data: 3 3 2 2 3 3 3 3 06/30 Flush 07/30 Write block 552 len 24576 data: 3 3 3 3 2 2 08/30 Read block 459 len 32768 09/30 Flush 10/30 Write block 287 len 16384 data: 3 3 3 3 11/30 Write block 322 len 12288 data: 3 3 2 12/30 Write block 154 len 8192 data: 4 3 13/30 Read block 168 len 28672 14/30 Flush 15/30 Write block 570 len 28672 data: 2 2 2 2 2 2 3 16/30 Read block 425 len 24576 17/30 Read block 36 len 20480 18/30 Write block 204 len 8192 data: 3 3 19/30 Read block 338 len 32768 20/30 Write block 156 len 24576 data: 3 3 3 3 3 2 21/30 Write block 427 len 40960 data: 3 3 3 3 4 3 3 3 2 2 22/30 Read block 98 len 40960 23/30 Read block 413 len 4096 24/30 Read block 89 len 12288 25/30 Write block 351 len 8192 data: 3 3 26/30 Read block 447 len 36864 27/30 Read block 563 len 24576 28/30 Read block 128 len 28672 29/30 Write block 523 len 40960 data: 4 4 3 3 2 2 2 2 2 2 30/30 Write block 434 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:53.978205466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 18 18 18 F F F 1 020-039 12 12 12 21 21 21 F F F 2 040-059 12 12 12 21 21 21 F F F 3 060-079 12 12 12 21 21 21 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 19 19 19 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 12 12 12 23 23 23 T T T 8 160-179 10 10 10 19 19 19 T T T 9 180-199 10 10 10 19 19 19 F F F 10 200-219 10 10 10 19 19 19 T T T 11 220-239 9 9 9 17 17 17 F F F 12 240-259 12 12 12 21 21 21 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 12 12 12 23 23 23 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 12 12 12 23 23 23 F F F 17 340-359 11 11 11 20 20 20 T T T 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 10 10 10 19 19 19 F F F 21 420-439 8 8 8 16 16 16 T T T 22 440-459 6 6 6 13 13 13 F F F 23 460-479 12 12 12 21 21 21 F F F 24 480-499 12 12 12 21 21 21 F F F 25 500-519 9 9 9 17 17 17 F F F 26 520-539 9 9 9 17 17 17 T T T 27 540-559 12 12 12 22 22 22 F F F 28 560-579 11 11 11 20 20 20 T T T 29 580-599 8 8 8 16 16 16 F F F Max gen: 12, Max flush: 23 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 18 18 18 F F F 1 020-039 12 12 12 21 21 21 F F F 2 040-059 12 12 12 21 21 21 F F F 3 060-079 12 12 12 21 21 21 F F F 4 080-099 10 10 10 18 18 18 F F F 5 100-119 10 10 10 19 19 19 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 13 13 13 24 24 24 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 9 9 9 17 17 17 F F F 12 240-259 12 12 12 21 21 21 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 12 12 12 23 23 23 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 12 12 12 23 23 23 F F F 17 340-359 13 13 13 24 24 24 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 10 10 10 19 19 19 F F F 21 420-439 13 13 13 24 24 24 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 12 12 12 21 21 21 F F F 24 480-499 12 12 12 21 21 21 F F F 25 500-519 9 9 9 17 17 17 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 13 13 13 24 24 24 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 13, Max flush: 24 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.227681592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.272304388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.272945082Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.272980548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.272999576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.273015305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} {"msg":"Crucible 2b188d12-ee04-4281-a0e1-8c7930d7baff has session id: a9fc531c-4d33-42e2-909e-4192c1f4f167","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.273516932Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Upstairs opts: Upstairs UUID: 2b188d12-ee04-4281-a0e1-8c7930d7baff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.273579746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Crucible stats registered with UUID: 2b188d12-ee04-4281-a0e1-8c7930d7baff","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.273599353Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:58.273622381Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274255565Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274281902Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} {"msg":"2b188d12-ee04-4281-a0e1-8c7930d7baff active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274342436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274360494Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"0","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274389051Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"0","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.2744032Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"1","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274417848Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"1","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274474612Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"2","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274516828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"2","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274544955Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274571042Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274608338Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"ds_connection connected from Ok(127.0.0.1:45915)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274636975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"ds_connection connected from Ok(127.0.0.1:47240)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.274664842Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"ds_connection connected from Ok(127.0.0.1:51512)","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.27469249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.275159901Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"1","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.275312955Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"0","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.27564504Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"2","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.275922021Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"1","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.275959797Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"0","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276238038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"2","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276482753Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276569434Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[0]R flush_numbers[0..12]: [18, 21, 21, 21, 18, 19, 16, 24, 24, 19, 24, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.27679833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[0]R generation[0..12]: [10, 12, 12, 12, 10, 10, 8, 13, 13, 10, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276818808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276833576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[1]R flush_numbers[0..12]: [18, 21, 21, 21, 18, 19, 16, 24, 24, 19, 24, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276847305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[1]R generation[0..12]: [10, 12, 12, 12, 10, 10, 8, 13, 13, 10, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276860593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276874212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[2]R flush_numbers[0..12]: [18, 21, 21, 21, 18, 19, 16, 24, 24, 19, 24, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276887761Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[2]R generation[0..12]: [10, 12, 12, 12, 10, 10, 8, 13, 13, 10, 13, 9]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276901339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276915158Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276927697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276940575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276953394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","Wait for a query_work_queue command to finish before sending IO ":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.276973172Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"":"downstairs","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.27698912Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277008668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"0","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277028426Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"1","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277042334Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"client":"2","":"downstairs","session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277055643Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277067892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"2b188d12-ee04-4281-a0e1-8c7930d7baff is now active with session: a9fc531c-4d33-42e2-909e-4192c1f4f167","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.27708057Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277092179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248,"session_id":"a9fc531c-4d33-42e2-909e-4192c1f4f167"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277104098Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} {"msg":"Activated sub_volume 2b188d12-ee04-4281-a0e1-8c7930d7baff","v":0,"name":"crucible","level":30,"time":"2025-03-18T04:59:58.277115157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 299 len 28672 data: 3 3 3 3 3 5 5 02/30 Read block 241 len 12288 03/30 Write block 228 len 24576 data: 2 2 3 3 3 3 04/30 Read block 393 len 20480 05/30 Read block 510 len 40960 06/30 Flush 07/30 Write block 117 len 12288 data: 2 2 2 08/30 Write block 173 len 28672 data: 4 3 3 3 2 3 3 09/30 Write block 84 len 4096 data: 3 10/30 Write block 256 len 24576 data: 4 3 2 4 4 4 11/30 Write block 390 len 12288 data: 2 2 2 12/30 Read block 103 len 8192 13/30 Flush 14/30 Write block 283 len 8192 data: 4 4 15/30 Write block 6 len 20480 data: 3 4 3 3 3 16/30 Write block 566 len 36864 data: 2 2 2 2 3 3 3 3 3 17/30 Write block 398 len 24576 data: 3 3 3 3 3 3 18/30 Read block 167 len 24576 19/30 Read block 578 len 24576 20/30 Read block 525 len 12288 21/30 Write block 544 len 20480 data: 2 3 3 3 3 22/30 Read block 317 len 32768 23/30 Write block 392 len 4096 data: 3 24/30 Write block 490 len 36864 data: 3 2 2 3 3 3 4 4 3 25/30 Read block 223 len 32768 26/30 Flush 27/30 Write block 54 len 36864 data: 2 2 3 3 3 3 3 3 2 28/30 Write block 572 len 4096 data: 4 29/30 Read block 91 len 32768 30/30 Write block 329 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T04:59:58.296293827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 12 12 12 21 21 21 F F F 2 040-059 12 12 12 21 21 21 T T T 3 060-079 12 12 12 21 21 21 T T T 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 14 14 14 26 26 26 F F F 13 260-279 14 14 14 26 26 26 F F F 14 280-299 14 14 14 27 27 27 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 12 12 12 23 23 23 T T T 17 340-359 13 13 13 24 24 24 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 14 14 14 27 27 27 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 13 13 13 24 24 24 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 12 12 12 21 21 21 F F F 24 480-499 14 14 14 27 27 27 F F F 25 500-519 9 9 9 17 17 17 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 14 14 14 27 27 27 T T T 29 580-599 8 8 8 16 16 16 F F F Max gen: 14, Max flush: 27 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 12 12 12 21 21 21 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 14 14 14 26 26 26 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 14 14 14 26 26 26 F F F 13 260-279 14 14 14 26 26 26 F F F 14 280-299 14 14 14 27 27 27 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 24 24 24 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 14 14 14 27 27 27 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 13 13 13 24 24 24 F F F 22 440-459 6 6 6 13 13 13 F F F 23 460-479 12 12 12 21 21 21 F F F 24 480-499 14 14 14 27 27 27 F F F 25 500-519 9 9 9 17 17 17 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 15 15 15 28 28 28 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 15, Max flush: 28 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.528655307Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:02.571784419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.572176548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.602970367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.603530439Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.603564645Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.603578783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.603593002Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} {"msg":"Crucible 51cc739f-6d66-4fc7-880f-0871b512bd70 has session id: ac243e41-faf1-4b8e-b11b-22ef10711422","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604326735Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Upstairs opts: Upstairs UUID: 51cc739f-6d66-4fc7-880f-0871b512bd70, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604363022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Crucible stats registered with UUID: 51cc739f-6d66-4fc7-880f-0871b512bd70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.60437906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:02.604393229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604751501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604777049Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} {"msg":"51cc739f-6d66-4fc7-880f-0871b512bd70 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604849451Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604877438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"0","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604908355Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"0","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604922923Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"1","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604937532Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"1","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.60495049Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"2","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604964429Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"2","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.604985507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605009064Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605032652Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"ds_connection connected from Ok(127.0.0.1:59929)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605056859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"ds_connection connected from Ok(127.0.0.1:47815)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605082866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"ds_connection connected from Ok(127.0.0.1:48463)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605108464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605875844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"1","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605919759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"0","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.605952836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"2","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.606525326Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"1","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.606646023Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"0","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.606675811Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"2","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607166229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607284517Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[0]R flush_numbers[0..12]: [27, 21, 28, 28, 26, 26, 16, 24, 26, 19, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607311034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[0]R generation[0..12]: [14, 12, 15, 15, 14, 14, 8, 13, 14, 10, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607327732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607342371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[1]R flush_numbers[0..12]: [27, 21, 28, 28, 26, 26, 16, 24, 26, 19, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.60735652Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[1]R generation[0..12]: [14, 12, 15, 15, 14, 14, 8, 13, 14, 10, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607370988Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607384786Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[2]R flush_numbers[0..12]: [27, 21, 28, 28, 26, 26, 16, 24, 26, 19, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607398185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[2]R generation[0..12]: [14, 12, 15, 15, 14, 14, 8, 13, 14, 10, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607416443Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607430762Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.60744373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607456699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607469218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607482186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"":"downstairs","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607495845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607507984Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"0Wait for a query_work_queue command to finish before sending IO ","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607529541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"1","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.60754356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"client":"2","":"downstairs","session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607556688Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607568907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"51cc739f-6d66-4fc7-880f-0871b512bd70 is now active with session: ac243e41-faf1-4b8e-b11b-22ef10711422","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607581656Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607593685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256,"session_id":"ac243e41-faf1-4b8e-b11b-22ef10711422"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607605574Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} {"msg":"Activated sub_volume 51cc739f-6d66-4fc7-880f-0871b512bd70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:02.607616762Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 439 len 8192 data: 2 2 02/30 Read block 148 len 12288 03/30 Read block 321 len 4096 04/30 Write block 192 len 20480 data: 3 3 3 3 4 05/30 Flush 06/30 Write block 173 len 40960 data: 5 4 4 4 3 4 4 2 2 2 07/30 Write block 510 len 24576 data: 4 4 3 3 4 3 08/30 Read block 275 len 32768 09/30 Write block 169 len 32768 data: 5 6 6 4 6 5 5 5 10/30 Read block 537 len 28672 11/30 Read block 411 len 12288 12/30 Read block 267 len 32768 13/30 Write block 443 len 20480 data: 2 2 2 2 2 14/30 Read block 546 len 4096 15/30 Read block 401 len 8192 16/30 Flush 17/30 Write block 241 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Read block 507 len 24576 19/30 Read block 44 len 32768 20/30 Read block 308 len 16384 21/30 Flush 22/30 Write block 574 len 32768 data: 4 3 4 4 4 3 2 2 23/30 Read block 403 len 20480 24/30 Write block 280 len 24576 data: 3 3 4 5 5 3 25/30 Flush 26/30 Write block 276 len 8192 data: 3 3 27/30 Read block 34 len 32768 28/30 Write block 166 len 24576 data: 5 5 5 6 7 7 29/30 Read block 281 len 12288 30/30 Write block 428 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 4 4 2 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:02.628694914Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 12 12 12 21 21 21 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 16 16 16 30 30 30 T T T 9 180-199 16 16 16 30 30 30 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 16 16 16 31 31 31 F F F 13 260-279 14 14 14 26 26 26 T T T 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 24 24 24 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 14 14 14 27 27 27 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 16 16 16 29 29 29 T T T 22 440-459 16 16 16 30 30 30 F F F 23 460-479 12 12 12 21 21 21 F F F 24 480-499 14 14 14 27 27 27 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 16 16 16 32 32 32 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 16, Max flush: 32 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 12 12 12 21 21 21 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 13 13 13 24 24 24 F F F 8 160-179 17 17 17 33 33 33 F F F 9 180-199 16 16 16 30 30 30 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 16 16 16 31 31 31 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 24 24 24 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 14 14 14 27 27 27 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 17 17 17 33 33 33 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 12 12 12 21 21 21 F F F 24 480-499 14 14 14 27 27 27 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 16 16 16 32 32 32 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 17, Max flush: 33 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.88948393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.933952163Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.934617943Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.93465427Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.934669868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.934684556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} {"msg":"Crucible 8de6d0eb-7e37-4971-9030-6f2bb8ead9c7 has session id: 56c71380-db19-4edd-bd0a-7c3f84939e31","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935192383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Upstairs opts: Upstairs UUID: 8de6d0eb-7e37-4971-9030-6f2bb8ead9c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935241168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Crucible stats registered with UUID: 8de6d0eb-7e37-4971-9030-6f2bb8ead9c7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935267586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:06.935288593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935832667Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935858254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} {"msg":"8de6d0eb-7e37-4971-9030-6f2bb8ead9c7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935925417Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935942675Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"0","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935971242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"0","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.935985511Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"1","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936001509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"1","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936024697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"2","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936051084Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"2","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936076871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936103219Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936122346Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:63876)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936249383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:47096)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936347223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:55227)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.936504917Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.937002745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"0","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.937040511Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"1","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.937066508Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"2","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.937746187Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"1","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.937788663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"0","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.937820459Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"2","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938379531Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938406948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[0]R flush_numbers[0..12]: [27, 21, 28, 28, 26, 26, 16, 24, 33, 30, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938435865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[0]R generation[0..12]: [14, 12, 15, 15, 14, 14, 8, 13, 17, 16, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938451294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938465782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[1]R flush_numbers[0..12]: [27, 21, 28, 28, 26, 26, 16, 24, 33, 30, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.93848307Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[1]R generation[0..12]: [14, 12, 15, 15, 14, 14, 8, 13, 17, 16, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938497099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938511507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[2]R flush_numbers[0..12]: [27, 21, 28, 28, 26, 26, 16, 24, 33, 30, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938525256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[2]R generation[0..12]: [14, 12, 15, 15, 14, 14, 8, 13, 17, 16, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938538794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938552363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938565042Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.93857812Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938590699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938604418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"":"downstairs","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938617596Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938629525Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"0","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938642664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"client":"1","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938655752Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","Wait for a query_work_queue command to finish before sending IO pid":1264,"client":"2","":"downstairs","session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.93867656Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938691738Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"8de6d0eb-7e37-4971-9030-6f2bb8ead9c7 is now active with session: 56c71380-db19-4edd-bd0a-7c3f84939e31","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938704767Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938716746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264,"session_id":"56c71380-db19-4edd-bd0a-7c3f84939e31"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938728805Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} {"msg":"Activated sub_volume 8de6d0eb-7e37-4971-9030-6f2bb8ead9c7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:06.938739704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 475 len 40960 data: 3 3 3 3 3 3 3 2 2 2 02/30 Read block 211 len 24576 03/30 Write block 432 len 12288 data: 4 4 5 04/30 Flush 05/30 Write block 353 len 12288 data: 3 3 3 06/30 Read block 393 len 24576 07/30 Read block 77 len 20480 08/30 Write block 9 len 16384 data: 4 4 3 3 09/30 Read block 193 len 36864 10/30 Read block 150 len 32768 11/30 Read block 345 len 24576 12/30 Read block 114 len 40960 13/30 Read block 356 len 4096 14/30 Read block 342 len 32768 15/30 Flush 16/30 Write block 136 len 4096 data: 3 17/30 Read block 6 len 40960 18/30 Flush 19/30 Write block 392 len 12288 data: 4 2 2 20/30 Read block 44 len 12288 21/30 Read block 558 len 24576 22/30 Read block 184 len 24576 23/30 Write block 393 len 28672 data: 3 3 2 2 2 4 4 24/30 Read block 139 len 12288 25/30 Write block 156 len 20480 data: 4 4 4 4 4 26/30 Read block 473 len 16384 27/30 Write block 28 len 4096 data: 2 28/30 Read block 473 len 20480 29/30 Read block 221 len 40960 30/30 Read block 223 len 40960 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:06.960137642Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 12 12 12 21 21 21 T T T 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 18 18 18 36 36 36 F F F 7 140-159 13 13 13 24 24 24 T T T 8 160-179 17 17 17 33 33 33 T T T 9 180-199 16 16 16 30 30 30 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 16 16 16 31 31 31 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 18 18 18 35 35 35 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 14 14 14 27 27 27 T T T 20 400-419 14 14 14 27 27 27 F F F 21 420-439 18 18 18 34 34 34 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 18 18 18 34 34 34 F F F 24 480-499 18 18 18 34 34 34 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 16 16 16 32 32 32 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 18, Max flush: 36 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 19 19 19 37 37 37 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 14 14 14 26 26 26 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 18 18 18 36 36 36 F F F 7 140-159 19 19 19 37 37 37 F F F 8 160-179 19 19 19 37 37 37 F F F 9 180-199 16 16 16 30 30 30 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 16 16 16 31 31 31 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 18 18 18 35 35 35 F F F 18 360-379 6 6 6 11 11 11 F F F 19 380-399 19 19 19 37 37 37 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 18 18 18 34 34 34 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 18 18 18 34 34 34 F F F 24 480-499 18 18 18 34 34 34 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 16 16 16 32 32 32 F F F 29 580-599 16 16 16 32 32 32 F F F Max gen: 19, Max flush: 37 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.18897555Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:11.231386067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.231761798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.262748746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.263424586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.263466202Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.26348322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.263501148Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} {"msg":"Crucible e8723575-8112-43d1-b34e-a98dfb3f27b5 has session id: 565596f1-00d8-452d-b5a8-2027b4856265","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264181387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Upstairs opts: Upstairs UUID: e8723575-8112-43d1-b34e-a98dfb3f27b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264234831Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Crucible stats registered with UUID: e8723575-8112-43d1-b34e-a98dfb3f27b5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264260089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:11.264281817Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264670966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264698343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} {"msg":"e8723575-8112-43d1-b34e-a98dfb3f27b5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264762136Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264780705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"0","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264808222Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"0","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.26482622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"1","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264840848Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"1","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264854907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"2","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264869316Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"2","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264882494Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.264896183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.26491942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"ds_connection connected from Ok(127.0.0.1:59671)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.26511513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"ds_connection connected from Ok(127.0.0.1:45852)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.265159135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"ds_connection connected from Ok(127.0.0.1:65131)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.265180733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.265806408Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"2","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.265849573Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"0","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.265913556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"1","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.266490046Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"2","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.266523413Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"1","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.266602995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"0","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267088814Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267200862Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[0]R flush_numbers[0..12]: [35, 37, 28, 28, 26, 26, 36, 37, 37, 30, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267266266Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[0]R generation[0..12]: [18, 19, 15, 15, 14, 14, 18, 19, 19, 16, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267296262Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267312601Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[1]R flush_numbers[0..12]: [35, 37, 28, 28, 26, 26, 36, 37, 37, 30, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267326669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[1]R generation[0..12]: [18, 19, 15, 15, 14, 14, 18, 19, 19, 16, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267340678Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267354636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[2]R flush_numbers[0..12]: [35, 37, 28, 28, 26, 26, 36, 37, 37, 30, 24, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267368415Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[2]R generation[0..12]: [18, 19, 15, 15, 14, 14, 18, 19, 19, 16, 13, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267382053Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267395982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267409851Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267437198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267462795Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267489632Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"":"downstairs","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267515889Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267541387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"0","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267568794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"1","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267596401Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"client":"2","":"downstairs","session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267623288Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267654485Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"e8723575-8112-43d1-b34e-a98dfb3f27b5 is now active with session: 565596f1-00d8-452d-b5a8-2027b4856265","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267674513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267686991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272,"session_id":"565596f1-00d8-452d-b5a8-2027b4856265"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.26769923Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} {"msg":"Activated sub_volume e8723575-8112-43d1-b34e-a98dfb3f27b5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:11.267710349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 20480 data: 5 5 5 6 3 02/30 Flush 03/30 Write block 471 len 12288 data: 2 2 2 04/30 Read block 56 len 24576 05/30 Read block 400 len 12288 06/30 Write block 237 len 4096 data: 2 07/30 Read block 163 len 16384 08/30 Read block 381 len 4096 09/30 Write block 134 len 32768 data: 2 2 4 3 3 3 2 2 10/30 Read block 116 len 20480 11/30 Write block 236 len 36864 data: 2 3 2 2 2 3 3 3 3 12/30 Write block 375 len 28672 data: 2 2 2 2 3 4 4 13/30 Write block 393 len 4096 data: 4 14/30 Read block 121 len 16384 15/30 Write block 476 len 40960 data: 4 4 4 4 4 4 3 3 3 2 16/30 Read block 391 len 24576 17/30 Write block 57 len 8192 data: 4 4 18/30 Read block 239 len 40960 19/30 Flush 20/30 Write block 352 len 32768 data: 4 4 4 4 3 3 2 2 21/30 Read block 547 len 24576 22/30 Read block 92 len 32768 23/30 Write block 324 len 20480 data: 3 3 3 4 4 24/30 Read block 524 len 8192 25/30 Write block 575 len 4096 data: 4 26/30 Read block 144 len 36864 27/30 Write block 581 len 32768 data: 3 2 2 3 3 3 4 4 28/30 Read block 111 len 28672 29/30 Write block 222 len 40960 data: 2 2 2 2 2 2 3 3 4 4 30/30 Write block 87 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:11.288037529Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 19 19 19 37 37 37 F F F 2 040-059 20 20 20 39 39 39 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 14 14 14 26 26 26 T T T 5 100-119 14 14 14 26 26 26 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 19 19 19 37 37 37 F F F 9 180-199 16 16 16 30 30 30 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 20 20 20 39 39 39 T T T 12 240-259 20 20 20 39 39 39 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 15 15 15 28 28 28 T T T 17 340-359 18 18 18 35 35 35 T T T 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 39 39 39 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 20 20 20 38 38 38 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 16 16 16 32 32 32 T T T 29 580-599 16 16 16 32 32 32 T T T Max gen: 20, Max flush: 39 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 F F F 1 020-039 19 19 19 37 37 37 F F F 2 040-059 20 20 20 39 39 39 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 21 21 21 40 40 40 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 19 19 19 37 37 37 F F F 9 180-199 16 16 16 30 30 30 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 21 21 21 40 40 40 F F F 12 240-259 20 20 20 39 39 39 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 21 21 21 40 40 40 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 39 39 39 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 20 20 20 38 38 38 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 20 20 20 39 39 39 F F F 25 500-519 16 16 16 30 30 30 F F F 26 520-539 13 13 13 24 24 24 F F F 27 540-559 14 14 14 27 27 27 F F F 28 560-579 21 21 21 40 40 40 F F F 29 580-599 21 21 21 40 40 40 F F F Max gen: 21, Max flush: 40 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.566205383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.612079209Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.612984015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.613019881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.61303467Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.613049138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} {"msg":"Crucible cc8c677a-8a78-4890-a78c-ec0742a099e4 has session id: 823758a3-7e42-499d-8524-0e6f96b724cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.613678092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Upstairs opts: Upstairs UUID: cc8c677a-8a78-4890-a78c-ec0742a099e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.613726647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Crucible stats registered with UUID: cc8c677a-8a78-4890-a78c-ec0742a099e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.613750185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:15.613764323Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614260491Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614285299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} {"msg":"cc8c677a-8a78-4890-a78c-ec0742a099e4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614351672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614377669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"0","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614407716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"0","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614423525Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614449692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614474989Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"2","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614493897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"2","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614515625Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614542252Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614570139Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43614)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614596636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:15.614698296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:15.614744091Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:15.614795546Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:63007)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.614834192Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.615012023Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.615390324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"2","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.61551728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"0","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.615964444Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"2","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.616072663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"0","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.616585359Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:15.616617296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.615702976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:35589)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.615785438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.616891932Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.617448885Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[0]R flush_numbers[0..12]: [35, 37, 39, 28, 40, 26, 39, 39, 37, 30, 24, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618032964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[0]R generation[0..12]: [18, 19, 20, 15, 21, 14, 20, 20, 19, 16, 13, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.61807186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618091918Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[1]R flush_numbers[0..12]: [35, 37, 39, 28, 40, 26, 39, 39, 37, 30, 24, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618108866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[1]R generation[0..12]: [18, 19, 20, 15, 21, 14, 20, 20, 19, 16, 13, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618125834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618143322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[2]R flush_numbers[0..12]: [35, 37, 39, 28, 40, 26, 39, 39, 37, 30, 24, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.61816039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[2]R generation[0..12]: [18, 19, 20, 15, 21, 14, 20, 20, 19, 16, 13, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618177238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618194447Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618211105Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618227403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618243222Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618266149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"":"downstairs","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618282007Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618301585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"0","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618318204Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"1","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618334362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"client":"2","":"downstairs","session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.6183506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618365859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"cc8c677a-8a78-4890-a78c-ec0742a099e4 is now active with session: 823758a3-7e42-499d-8524-0e6f96b724cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618381887Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618396606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280,"session_id":"823758a3-7e42-499d-8524-0e6f96b724cb"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618411604Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} {"msg":"Activated sub_volume cc8c677a-8a78-4890-a78c-ec0742a099e4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:25.618425313Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 176 len 4096 data: 6 02/30 Write block 536 len 32768 data: 3 3 3 2 2 2 2 2 03/30 Write block 330 len 16384 data: 5 5 4 3 04/30 Read block 329 len 16384 05/30 Write block 509 len 4096 data: 3 06/30 Write block 331 len 8192 data: 6 5 07/30 Write block 584 len 16384 data: 4 4 4 5 08/30 Read block 69 len 40960 09/30 Write block 184 len 4096 data: 2 10/30 Write block 424 len 8192 data: 2 2 11/30 Write block 73 len 28672 data: 2 2 2 2 2 2 3 12/30 Read block 505 len 16384 13/30 Flush 14/30 Write block 485 len 24576 data: 3 3 3 3 3 4 15/30 Read block 591 len 24576 16/30 Write block 244 len 36864 data: 4 3 3 3 3 4 4 3 3 17/30 Read block 552 len 4096 18/30 Write block 504 len 4096 data: 3 19/30 Read block 531 len 28672 20/30 Write block 235 len 20480 data: 3 3 4 3 3 21/30 Read block 459 len 12288 22/30 Flush 23/30 Write block 59 len 8192 data: 4 4 24/30 Read block 445 len 16384 25/30 Write block 0 len 32768 data: 2 2 2 2 3 3 4 5 26/30 Read block 541 len 36864 27/30 Write block 449 len 36864 data: 2 2 2 2 2 3 3 3 3 28/30 Write block 81 len 4096 data: 4 29/30 Write block 391 len 40960 data: 3 5 5 4 3 3 3 5 5 4 30/30 Read block 302 len 16384 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:25.637840398Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 35 35 35 T T T 1 020-039 19 19 19 37 37 37 F F F 2 040-059 20 20 20 39 39 39 T T T 3 060-079 22 22 22 41 41 41 T T T 4 080-099 21 21 21 40 40 40 T T T 5 100-119 14 14 14 26 26 26 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 22 22 22 41 41 41 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 22 22 22 42 42 42 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 22 22 22 41 41 41 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 20 20 20 39 39 39 T T T 20 400-419 14 14 14 27 27 27 T T T 21 420-439 22 22 22 41 41 41 F F F 22 440-459 16 16 16 30 30 30 T T T 23 460-479 20 20 20 39 39 39 F F F 24 480-499 22 22 22 42 42 42 F F F 25 500-519 22 22 22 42 42 42 F F F 26 520-539 22 22 22 41 41 41 F F F 27 540-559 22 22 22 41 41 41 F F F 28 560-579 21 21 21 40 40 40 F F F 29 580-599 22 22 22 41 41 41 F F F Max gen: 22, Max flush: 42 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 43 43 43 F F F 1 020-039 19 19 19 37 37 37 F F F 2 040-059 23 23 23 43 43 43 F F F 3 060-079 23 23 23 43 43 43 F F F 4 080-099 23 23 23 43 43 43 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 22 22 22 41 41 41 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 13 13 13 24 24 24 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 22 22 22 42 42 42 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 16 16 16 32 32 32 F F F 15 300-319 14 14 14 25 25 25 F F F 16 320-339 22 22 22 41 41 41 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 23 23 23 43 43 43 F F F 20 400-419 23 23 23 43 43 43 F F F 21 420-439 22 22 22 41 41 41 F F F 22 440-459 23 23 23 43 43 43 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 22 22 22 42 42 42 F F F 25 500-519 22 22 22 42 42 42 F F F 26 520-539 22 22 22 41 41 41 F F F 27 540-559 22 22 22 41 41 41 F F F 28 560-579 21 21 21 40 40 40 F F F 29 580-599 22 22 22 41 41 41 F F F Max gen: 23, Max flush: 43 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.889334423Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.934468137Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.935135527Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.935171983Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.935187172Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.93520134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} {"msg":"Crucible af598a38-2c6a-40c6-9576-6410bd1548aa has session id: 352e3dcf-a814-433a-beeb-7aebcfc862ae","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.935806787Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Upstairs opts: Upstairs UUID: af598a38-2c6a-40c6-9576-6410bd1548aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.935853662Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Crucible stats registered with UUID: af598a38-2c6a-40c6-9576-6410bd1548aa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.93587349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:29.935887759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936342021Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936377068Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} {"msg":"af598a38-2c6a-40c6-9576-6410bd1548aa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936431742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936461879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"0","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936493585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"0","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936509334Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"1","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936524292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"1","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936537691Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"2","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936551719Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"2","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936564878Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936578696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936602624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:64896)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936630271Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:61656)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936657638Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:59161)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.936911942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.937338947Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"0","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.937382383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"1","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.937421069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"2","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938029755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"1","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938064921Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"2","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938131974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"0","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.93865477Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938779717Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[0]R flush_numbers[0..12]: [43, 37, 43, 43, 43, 26, 39, 39, 41, 41, 24, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.93884848Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[0]R generation[0..12]: [23, 19, 23, 23, 23, 14, 20, 20, 22, 22, 13, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938869098Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938885086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[1]R flush_numbers[0..12]: [43, 37, 43, 43, 43, 26, 39, 39, 41, 41, 24, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938899874Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[1]R generation[0..12]: [23, 19, 23, 23, 23, 14, 20, 20, 22, 22, 13, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938913373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938927102Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[2]R flush_numbers[0..12]: [43, 37, 43, 43, 43, 26, 39, 39, 41, 41, 24, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.93894055Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[2]R generation[0..12]: [23, 19, 23, 23, 23, 14, 20, 20, 22, 22, 13, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938956679Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938983506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.938999324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939012543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939025171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.9390379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"":"downstairs","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939057368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939069627Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"0","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939082775Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"1","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939095794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"client":"2","":"downstairs","session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939108723Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939120871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"af598a38-2c6a-40c6-9576-6410bd1548aa is now active with session: 352e3dcf-a814-433a-beeb-7aebcfc862ae","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.93913366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939145559Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288,"session_id":"352e3dcf-a814-433a-beeb-7aebcfc862ae"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939157598Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} {"msg":"Activated sub_volume af598a38-2c6a-40c6-9576-6410bd1548aa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:29.939168646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 295 len 36864 data: 2 2 2 3 4 4 4 4 4 02/30 Flush 03/30 Write block 43 len 40960 data: 3 3 3 3 3 3 3 3 3 3 04/30 Read block 35 len 40960 05/30 Read block 559 len 8192 06/30 Read block 574 len 4096 07/30 Write block 464 len 8192 data: 2 2 08/30 Read block 459 len 4096 09/30 Write block 417 len 12288 data: 3 3 3 10/30 Write block 520 len 16384 data: 4 3 4 5 11/30 Write block 206 len 36864 data: 3 3 3 2 2 2 2 3 3 12/30 Read block 465 len 40960 13/30 Read block 121 len 4096 14/30 Read block 297 len 4096 15/30 Write block 95 len 32768 data: 2 2 2 2 2 2 2 3 16/30 Read block 310 len 28672 17/30 Write block 579 len 24576 data: 4 3 4 3 3 5 18/30 Read block 135 len 12288 19/30 Write block 252 len 4096 data: 4 20/30 Write block 22 len 24576 data: 3 3 3 3 2 2 21/30 Write block 531 len 4096 data: 3 22/30 Flush 23/30 Write block 417 len 20480 data: 4 4 4 2 2 24/30 Write block 241 len 36864 data: 4 4 4 5 4 4 4 4 5 25/30 Read block 188 len 28672 26/30 Write block 287 len 32768 data: 4 4 4 4 2 2 2 2 27/30 Write block 389 len 8192 data: 2 3 28/30 Read block 379 len 16384 29/30 Read block 535 len 16384 30/30 Read block 48 len 20480 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:29.959375229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 43 43 43 F F F 1 020-039 24 24 24 45 45 45 F F F 2 040-059 24 24 24 45 45 45 F F F 3 060-079 23 23 23 43 43 43 F F F 4 080-099 24 24 24 45 45 45 F F F 5 100-119 24 24 24 45 45 45 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 22 22 22 41 41 41 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 24 24 24 45 45 45 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 24 24 24 45 45 45 T T T 13 260-279 17 17 17 33 33 33 F F F 14 280-299 24 24 24 44 44 44 T T T 15 300-319 24 24 24 44 44 44 F F F 16 320-339 22 22 22 41 41 41 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 23 23 23 43 43 43 T T T 20 400-419 24 24 24 45 45 45 T T T 21 420-439 22 22 22 41 41 41 T T T 22 440-459 23 23 23 43 43 43 F F F 23 460-479 24 24 24 45 45 45 F F F 24 480-499 22 22 22 42 42 42 F F F 25 500-519 22 22 22 42 42 42 F F F 26 520-539 24 24 24 45 45 45 F F F 27 540-559 22 22 22 41 41 41 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 24, Max flush: 45 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 43 43 43 F F F 1 020-039 24 24 24 45 45 45 F F F 2 040-059 24 24 24 45 45 45 F F F 3 060-079 23 23 23 43 43 43 F F F 4 080-099 24 24 24 45 45 45 F F F 5 100-119 24 24 24 45 45 45 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 22 22 22 41 41 41 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 24 24 24 45 45 45 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 25 25 25 46 46 46 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 25 25 25 46 46 46 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 22 22 22 41 41 41 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 25 25 25 46 46 46 F F F 20 400-419 25 25 25 46 46 46 F F F 21 420-439 25 25 25 46 46 46 F F F 22 440-459 23 23 23 43 43 43 F F F 23 460-479 24 24 24 45 45 45 F F F 24 480-499 22 22 22 42 42 42 F F F 25 500-519 22 22 22 42 42 42 F F F 26 520-539 24 24 24 45 45 45 F F F 27 540-559 22 22 22 41 41 41 F F F 28 560-579 24 24 24 45 45 45 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 25, Max flush: 46 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.197457664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24180054Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.242459551Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.242494308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.242508276Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.242522534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} {"msg":"Crucible da3c55d4-cc0a-4cb4-a62d-c8bbaee2d803 has session id: dceb96ea-f730-458d-9d59-17974e3b1b69","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243101964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Upstairs opts: Upstairs UUID: da3c55d4-cc0a-4cb4-a62d-c8bbaee2d803, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24314536Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Crucible stats registered with UUID: da3c55d4-cc0a-4cb4-a62d-c8bbaee2d803","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243162658Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:34.243175777Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243652837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243687173Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} {"msg":"da3c55d4-cc0a-4cb4-a62d-c8bbaee2d803 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243722009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243737778Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"0","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243778974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"0","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243806611Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"1","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243832418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"1","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243858685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"2","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243885562Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"2","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24391193Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243939867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.243967924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"ds_connection connected from Ok(127.0.0.1:56572)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24400077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"ds_connection connected from Ok(127.0.0.1:58749)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.244030577Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"ds_connection connected from Ok(127.0.0.1:50406)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.244056185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.244708837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"1","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.244754152Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"0","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.245197635Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"2","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.245474387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"0","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.245504654Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"1","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.245926619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"2","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24611736Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24620729Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[0]R flush_numbers[0..12]: [43, 45, 45, 43, 45, 45, 39, 39, 41, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246477522Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[0]R generation[0..12]: [23, 24, 24, 23, 24, 24, 20, 20, 22, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246502039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246517268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[1]R flush_numbers[0..12]: [43, 45, 45, 43, 45, 45, 39, 39, 41, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246535356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[1]R generation[0..12]: [23, 24, 24, 23, 24, 24, 20, 20, 22, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246549625Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246564183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[2]R flush_numbers[0..12]: [43, 45, 45, 43, 45, 45, 39, 39, 41, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246577972Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[2]R generation[0..12]: [23, 24, 24, 23, 24, 24, 20, 20, 22, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24659153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246605589Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246618537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246631456Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246644175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246656853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"":"downstairs","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246669942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246681891Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"0","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:00:34.246695339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"1","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246715677Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"client":"2","":"downstairs","session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246729006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246743174Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"da3c55d4-cc0a-4cb4-a62d-c8bbaee2d803 is now active with session: dceb96ea-f730-458d-9d59-17974e3b1b69","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246756223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246768092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296,"session_id":"dceb96ea-f730-458d-9d59-17974e3b1b69"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.24678009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} {"msg":"Activated sub_volume da3c55d4-cc0a-4cb4-a62d-c8bbaee2d803","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:34.246791249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 445 len 12288 data: 3 3 3 02/30 Read block 326 len 28672 03/30 Read block 98 len 32768 04/30 Write block 11 len 24576 data: 4 4 3 3 3 3 05/30 Write block 388 len 16384 data: 2 3 4 4 06/30 Write block 425 len 24576 data: 3 3 4 5 5 4 07/30 Write block 501 len 28672 data: 4 5 4 4 2 2 2 08/30 Write block 457 len 4096 data: 4 09/30 Write block 171 len 20480 data: 8 5 7 6 6 10/30 Write block 78 len 12288 data: 3 4 3 11/30 Read block 352 len 4096 12/30 Write block 431 len 36864 data: 6 6 6 7 4 3 2 2 3 13/30 Write block 291 len 12288 data: 3 3 3 14/30 Write block 55 len 40960 data: 3 4 5 5 5 5 4 3 3 3 15/30 Read block 320 len 20480 16/30 Read block 214 len 20480 17/30 Read block 264 len 8192 18/30 Read block 292 len 24576 19/30 Flush 20/30 Write block 74 len 8192 data: 3 3 21/30 Write block 486 len 16384 data: 4 4 4 4 22/30 Flush 23/30 Write block 477 len 40960 data: 5 5 5 5 5 4 4 4 4 5 24/30 Write block 143 len 36864 data: 2 2 2 2 3 3 3 3 3 25/30 Write block 527 len 12288 data: 3 3 3 26/30 Write block 241 len 12288 data: 5 5 5 27/30 Write block 6 len 28672 data: 5 6 4 5 5 5 5 28/30 Write block 535 len 32768 data: 3 4 4 4 3 3 3 3 29/30 Write block 390 len 12288 data: 5 5 6 30/30 Write block 559 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:34.2650035Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 47 47 47 T T T 1 020-039 24 24 24 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 26 26 26 48 48 48 F F F 4 080-099 26 26 26 47 47 47 F F F 5 100-119 24 24 24 45 45 45 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 20 20 20 39 39 39 T T T 8 160-179 26 26 26 47 47 47 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 24 24 24 45 45 45 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 25 25 25 46 46 46 T T T 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 22 22 22 41 41 41 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 26 26 26 47 47 47 T T T 20 400-419 25 25 25 46 46 46 F F F 21 420-439 26 26 26 47 47 47 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 24 24 24 45 45 45 T T T 24 480-499 26 26 26 48 48 48 T T T 25 500-519 26 26 26 47 47 47 F F F 26 520-539 24 24 24 45 45 45 T T T 27 540-559 22 22 22 41 41 41 T T T 28 560-579 24 24 24 45 45 45 T T T 29 580-599 24 24 24 45 45 45 F F F Max gen: 26, Max flush: 48 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 49 49 49 F F F 1 020-039 24 24 24 45 45 45 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 26 26 26 48 48 48 F F F 4 080-099 26 26 26 47 47 47 F F F 5 100-119 24 24 24 45 45 45 F F F 6 120-139 20 20 20 39 39 39 F F F 7 140-159 27 27 27 49 49 49 F F F 8 160-179 26 26 26 47 47 47 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 24 24 24 45 45 45 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 27 27 27 49 49 49 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 22 22 22 41 41 41 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 27 27 27 49 49 49 F F F 20 400-419 25 25 25 46 46 46 F F F 21 420-439 26 26 26 47 47 47 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 27 27 27 49 49 49 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 27 27 27 49 49 49 F F F 27 540-559 27 27 27 49 49 49 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 27, Max flush: 49 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.52220958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.567156342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.567819623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.567854729Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.567868928Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.567884226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} {"msg":"Crucible e418ba66-4213-4d3d-9130-54504dcba339 has session id: 50c78def-21f5-410e-b790-2917fe622c84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.568576554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Upstairs opts: Upstairs UUID: e418ba66-4213-4d3d-9130-54504dcba339, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.56861259Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Crucible stats registered with UUID: e418ba66-4213-4d3d-9130-54504dcba339","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.568627739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:38.568640418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569075112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.56909997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} {"msg":"e418ba66-4213-4d3d-9130-54504dcba339 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569165503Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569182801Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"0","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569211378Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"0","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569225406Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"1","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569244464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"1","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.56928791Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"2","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569328246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"2","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569356833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.56938518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569414187Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"ds_connection connected from Ok(127.0.0.1:50450)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569441254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"ds_connection connected from Ok(127.0.0.1:50497)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569468841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"ds_connection connected from Ok(127.0.0.1:45637)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.569531874Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.570094896Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"0","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.57024244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"1","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.570687634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"0","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.57082393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"1","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.570852967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"2","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.571282402Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.571368623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.571573262Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"2","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[0]R flush_numbers[0..12]: [49, 45, 47, 48, 47, 45, 39, 49, 47, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572136203Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[0]R generation[0..12]: [27, 24, 26, 26, 26, 24, 20, 27, 26, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572157401Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572172429Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[1]R flush_numbers[0..12]: [49, 45, 47, 48, 47, 45, 39, 49, 47, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572186188Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[1]R generation[0..12]: [27, 24, 26, 26, 26, 24, 20, 27, 26, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572199766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572213485Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[2]R flush_numbers[0..12]: [49, 45, 47, 48, 47, 45, 39, 49, 47, 41, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572227304Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[2]R generation[0..12]: [27, 24, 26, 26, 26, 24, 20, 27, 26, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572240722Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572254521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572267379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572280258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572301756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572315174Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"":"downstairs","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572328283Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572341481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"0","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.57235481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"1","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572367959Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"client":"2","":"downstairs","session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572381487Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572393706Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"e418ba66-4213-4d3d-9130-54504dcba339 is now active with session: 50c78def-21f5-410e-b790-2917fe622c84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572406365Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572418084Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304,"session_id":"50c78def-21f5-410e-b790-2917fe622c84"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572430192Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} {"msg":"Activated sub_volume e418ba66-4213-4d3d-9130-54504dcba339","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:38.572441491Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 171 len 28672 data: 9 6 8 7 7 7 4 02/30 Read block 440 len 12288 03/30 Read block 240 len 36864 04/30 Read block 479 len 20480 05/30 Write block 119 len 8192 data: 3 2 06/30 Write block 182 len 40960 data: 3 2 3 2 2 2 2 2 2 3 07/30 Flush 08/30 Write block 30 len 32768 data: 3 3 3 3 3 3 3 3 09/30 Flush 10/30 Write block 409 len 8192 data: 2 3 11/30 Read block 106 len 8192 12/30 Read block 457 len 8192 13/30 Flush 14/30 Write block 15 len 20480 data: 4 4 3 3 3 15/30 Read block 129 len 28672 16/30 Write block 324 len 24576 data: 4 4 4 5 5 6 17/30 Read block 488 len 4096 18/30 Read block 224 len 8192 19/30 Flush 20/30 Write block 74 len 8192 data: 4 4 21/30 Write block 460 len 16384 data: 3 3 3 2 22/30 Read block 253 len 16384 23/30 Read block 237 len 20480 24/30 Read block 192 len 4096 25/30 Read block 11 len 12288 26/30 Read block 139 len 24576 27/30 Write block 122 len 36864 data: 2 2 2 3 3 3 3 3 3 28/30 Read block 187 len 12288 29/30 Write block 65 len 32768 data: 3 2 2 2 2 2 2 2 30/30 Read block 426 len 24576 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 10 11 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:38.593181808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 53 53 53 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 26 26 26 48 48 48 T T T 4 080-099 26 26 26 47 47 47 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 28 28 28 50 50 50 T T T 7 140-159 27 27 27 49 49 49 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 28 28 28 50 50 50 F F F 10 200-219 24 24 24 45 45 45 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 27 27 27 49 49 49 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 28 28 28 53 53 53 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 27 27 27 49 49 49 F F F 20 400-419 28 28 28 52 52 52 F F F 21 420-439 26 26 26 47 47 47 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 27 27 27 49 49 49 T T T 24 480-499 27 27 27 49 49 49 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 27 27 27 49 49 49 F F F 27 540-559 27 27 27 49 49 49 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 28, Max flush: 53 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 53 53 53 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 29 29 29 54 54 54 F F F 4 080-099 26 26 26 47 47 47 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 29 29 29 54 54 54 F F F 7 140-159 27 27 27 49 49 49 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 28 28 28 50 50 50 F F F 10 200-219 24 24 24 45 45 45 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 27 27 27 49 49 49 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 28 28 28 53 53 53 F F F 17 340-359 21 21 21 40 40 40 F F F 18 360-379 20 20 20 39 39 39 F F F 19 380-399 27 27 27 49 49 49 F F F 20 400-419 28 28 28 52 52 52 F F F 21 420-439 26 26 26 47 47 47 F F F 22 440-459 26 26 26 47 47 47 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 26 26 26 47 47 47 F F F 26 520-539 27 27 27 49 49 49 F F F 27 540-559 27 27 27 49 49 49 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 29, Max flush: 54 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.824385799Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:42.869295606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.869714242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.9009001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.901546003Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.901588838Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.901607046Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.901624975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} {"msg":"Crucible 5b77330b-a72b-4ba2-bbc1-5402883f7436 has session id: ab302df2-f353-4588-b422-acba3a1f7878","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902128402Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Upstairs opts: Upstairs UUID: 5b77330b-a72b-4ba2-bbc1-5402883f7436, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902184956Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Crucible stats registered with UUID: 5b77330b-a72b-4ba2-bbc1-5402883f7436","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902222662Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:42.90224418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902613741Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902636289Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} {"msg":"5b77330b-a72b-4ba2-bbc1-5402883f7436 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902692863Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.90271961Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"0","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902747938Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"0","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902762256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"1","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902776515Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"1","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902789663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"2","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902803412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"2","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.90281631Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902829659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.902847687Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"ds_connection connected from Ok(127.0.0.1:55468)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.903066384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"ds_connection connected from Ok(127.0.0.1:59922)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.903169494Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"ds_connection connected from Ok(127.0.0.1:61042)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.90319968Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.903733525Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"2","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.903764641Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"1","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.903863621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"0","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.904467498Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"1","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.904496445Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"2","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.904512203Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"0","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905051147Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905158716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[0]R flush_numbers[0..12]: [53, 51, 47, 54, 47, 50, 54, 49, 50, 50, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905258006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[0]R generation[0..12]: [28, 28, 26, 29, 26, 28, 29, 27, 28, 28, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905278074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905293742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[1]R flush_numbers[0..12]: [53, 51, 47, 54, 47, 50, 54, 49, 50, 50, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905307531Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[1]R generation[0..12]: [28, 28, 26, 29, 26, 28, 29, 27, 28, 28, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905320859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905339167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[2]R flush_numbers[0..12]: [53, 51, 47, 54, 47, 50, 54, 49, 50, 50, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905353446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[2]R generation[0..12]: [28, 28, 26, 29, 26, 28, 29, 27, 28, 28, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905366884Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905380643Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905394162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.90540709Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905419949Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905432807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"":"downstairs","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905445896Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905457725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"0","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:00:42.905471394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"1","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905490652Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"client":"2","":"downstairs","session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.90550369Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905515759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"5b77330b-a72b-4ba2-bbc1-5402883f7436 is now active with session: ab302df2-f353-4588-b422-acba3a1f7878","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905528448Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905543416Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312,"session_id":"ab302df2-f353-4588-b422-acba3a1f7878"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905555815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} {"msg":"Activated sub_volume 5b77330b-a72b-4ba2-bbc1-5402883f7436","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:42.905567224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 513 len 4096 data: 4 02/30 Read block 125 len 32768 03/30 Write block 383 len 24576 data: 3 3 3 3 2 3 04/30 Write block 125 len 16384 data: 4 4 4 4 05/30 Write block 204 len 28672 data: 4 4 4 4 4 3 3 06/30 Read block 563 len 24576 07/30 Write block 68 len 36864 data: 3 3 3 3 3 3 5 5 3 08/30 Flush 09/30 Write block 554 len 20480 data: 4 4 3 3 2 10/30 Read block 267 len 8192 11/30 Write block 353 len 40960 data: 5 5 5 4 4 3 3 3 3 3 12/30 Write block 211 len 4096 data: 3 13/30 Read block 244 len 36864 14/30 Read block 587 len 16384 15/30 Read block 477 len 4096 16/30 Write block 11 len 32768 data: 6 6 4 4 5 5 4 4 17/30 Write block 311 len 24576 data: 4 3 2 2 2 3 18/30 Read block 297 len 16384 19/30 Read block 568 len 8192 20/30 Read block 8 len 32768 21/30 Read block 105 len 8192 22/30 Read block 23 len 16384 23/30 Read block 578 len 4096 24/30 Write block 131 len 20480 data: 2 2 2 3 3 25/30 Read block 197 len 12288 26/30 Write block 438 len 16384 data: 3 4 3 2 27/30 Write block 54 len 40960 data: 3 4 5 6 6 6 6 5 4 4 28/30 Read block 106 len 4096 29/30 Write block 432 len 4096 data: 7 30/30 Read block 165 len 16384 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 5 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 22 21 21 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:42.925484587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 53 53 53 T T T 1 020-039 28 28 28 51 51 51 F F F 2 040-059 26 26 26 47 47 47 T T T 3 060-079 30 30 30 55 55 55 T T T 4 080-099 26 26 26 47 47 47 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 30 30 30 55 55 55 T T T 7 140-159 27 27 27 49 49 49 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 28 28 28 50 50 50 F F F 10 200-219 30 30 30 55 55 55 T T T 11 220-239 22 22 22 42 42 42 F F F 12 240-259 27 27 27 49 49 49 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 24 24 24 44 44 44 T T T 16 320-339 28 28 28 53 53 53 F F F 17 340-359 21 21 21 40 40 40 T T T 18 360-379 20 20 20 39 39 39 T T T 19 380-399 30 30 30 55 55 55 F F F 20 400-419 28 28 28 52 52 52 F F F 21 420-439 26 26 26 47 47 47 T T T 22 440-459 26 26 26 47 47 47 T T T 23 460-479 29 29 29 54 54 54 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 27 27 27 49 49 49 F F F 27 540-559 27 27 27 49 49 49 T T T 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 30, Max flush: 55 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 56 56 56 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 31 31 31 56 56 56 F F F 3 060-079 31 31 31 56 56 56 F F F 4 080-099 26 26 26 47 47 47 F F F 5 100-119 28 28 28 50 50 50 F F F 6 120-139 31 31 31 56 56 56 F F F 7 140-159 27 27 27 49 49 49 F F F 8 160-179 28 28 28 50 50 50 F F F 9 180-199 28 28 28 50 50 50 F F F 10 200-219 31 31 31 56 56 56 F F F 11 220-239 22 22 22 42 42 42 F F F 12 240-259 27 27 27 49 49 49 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 31 31 31 56 56 56 F F F 16 320-339 28 28 28 53 53 53 F F F 17 340-359 31 31 31 56 56 56 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 30 30 30 55 55 55 F F F 20 400-419 28 28 28 52 52 52 F F F 21 420-439 31 31 31 56 56 56 F F F 22 440-459 31 31 31 56 56 56 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 27 27 27 49 49 49 F F F 27 540-559 31 31 31 56 56 56 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 31, Max flush: 56 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.219556182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:47.262274077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.262659917Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.293467814Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.294116506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.294156992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.29417418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.294192668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} {"msg":"Crucible 2678b32f-cf1c-474e-85a5-0f1d97241c24 has session id: 8a849536-9380-43b1-be80-a2a6bd9f5598","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.294864638Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Upstairs opts: Upstairs UUID: 2678b32f-cf1c-474e-85a5-0f1d97241c24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.294909993Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Crucible stats registered with UUID: 2678b32f-cf1c-474e-85a5-0f1d97241c24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.294927572Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:47.29494038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.29533028Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295353957Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} {"msg":"2678b32f-cf1c-474e-85a5-0f1d97241c24 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295440438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295458846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"0","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295490253Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"0","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295504311Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"1","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295529799Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"1","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295554626Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"2","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295576234Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"2","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295602001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295629049Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295656226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"ds_connection connected from Ok(127.0.0.1:47501)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295843756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"ds_connection connected from Ok(127.0.0.1:54914)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295885552Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"ds_connection connected from Ok(127.0.0.1:33687)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.295917548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.296557872Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"1","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.296586499Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"2","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.296602577Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"0","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.29724638Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"2","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.297276227Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"1","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.297292645Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"0","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.2978247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.297929758Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[0]R flush_numbers[0..12]: [56, 51, 56, 56, 47, 50, 56, 49, 50, 50, 56, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.297998841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[0]R generation[0..12]: [31, 28, 31, 31, 26, 28, 31, 27, 28, 28, 31, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298018579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298033848Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[1]R flush_numbers[0..12]: [56, 51, 56, 56, 47, 50, 56, 49, 50, 50, 56, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298048366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[1]R generation[0..12]: [31, 28, 31, 31, 26, 28, 31, 27, 28, 28, 31, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298061855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298075653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[2]R flush_numbers[0..12]: [56, 51, 56, 56, 47, 50, 56, 49, 50, 50, 56, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298088862Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[2]R generation[0..12]: [31, 28, 31, 31, 26, 28, 31, 27, 28, 28, 31, 22]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298102031Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298116529Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298134097Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298147706Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298160565Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298173293Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":Wait for a query_work_queue command to finish before sending IO 1320,"":"downstairs","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298193981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.29820607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"0","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298219488Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"1","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298232527Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"client":"2","":"downstairs","session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298245786Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298257774Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"2678b32f-cf1c-474e-85a5-0f1d97241c24 is now active with session: 8a849536-9380-43b1-be80-a2a6bd9f5598","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298270513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298282272Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320,"session_id":"8a849536-9380-43b1-be80-a2a6bd9f5598"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298294211Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} {"msg":"Activated sub_volume 2678b32f-cf1c-474e-85a5-0f1d97241c24","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:47.298305269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 12288 data: 6 7 8 02/30 Write block 187 len 20480 data: 3 3 3 3 4 03/30 Read block 28 len 8192 04/30 Read block 394 len 40960 05/30 Read block 383 len 36864 06/30 Write block 334 len 36864 data: 3 3 3 3 3 2 2 2 3 07/30 Read block 16 len 20480 08/30 Read block 390 len 24576 09/30 Write block 522 len 36864 data: 5 6 5 4 4 4 4 4 3 10/30 Read block 562 len 4096 11/30 Write block 216 len 28672 data: 3 3 3 3 3 2 3 12/30 Read block 194 len 8192 13/30 Read block 94 len 40960 14/30 Write block 83 len 20480 data: 4 4 3 2 3 15/30 Read block 296 len 28672 16/30 Write block 418 len 8192 data: 5 5 17/30 Read block 92 len 36864 18/30 Read block 98 len 8192 19/30 Write block 158 len 28672 data: 5 5 5 3 2 3 3 20/30 Read block 183 len 20480 21/30 Write block 399 len 4096 data: 6 22/30 Read block 53 len 36864 23/30 Write block 102 len 32768 data: 4 2 2 2 2 2 2 2 24/30 Read block 34 len 4096 25/30 Read block 500 len 32768 26/30 Write block 76 len 8192 data: 4 3 27/30 Write block 449 len 20480 data: 3 3 3 3 3 28/30 Write block 241 len 28672 data: 6 6 6 6 5 5 5 29/30 Read block 74 len 16384 30/30 Read block 429 len 32768 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 29 30 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:47.319570212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 56 56 56 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 31 31 31 56 56 56 F F F 3 060-079 31 31 31 56 56 56 T T T 4 080-099 26 26 26 47 47 47 T T T 5 100-119 28 28 28 50 50 50 T T T 6 120-139 31 31 31 56 56 56 F F F 7 140-159 27 27 27 49 49 49 T T T 8 160-179 28 28 28 50 50 50 T T T 9 180-199 28 28 28 50 50 50 T T T 10 200-219 31 31 31 56 56 56 T T T 11 220-239 22 22 22 42 42 42 T T T 12 240-259 27 27 27 49 49 49 T T T 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 31 31 31 56 56 56 F F F 16 320-339 28 28 28 53 53 53 T T T 17 340-359 31 31 31 56 56 56 T T T 18 360-379 31 31 31 56 56 56 F F F 19 380-399 30 30 30 55 55 55 T T T 20 400-419 28 28 28 52 52 52 T T T 21 420-439 31 31 31 56 56 56 F F F 22 440-459 31 31 31 56 56 56 T T T 23 460-479 29 29 29 54 54 54 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 27 27 27 49 49 49 T T T 27 540-559 31 31 31 56 56 56 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 31, Max flush: 56 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 56 56 56 F F F 1 020-039 28 28 28 51 51 51 F F F 2 040-059 31 31 31 56 56 56 F F F 3 060-079 32 32 32 57 57 57 F F F 4 080-099 32 32 32 57 57 57 F F F 5 100-119 32 32 32 57 57 57 F F F 6 120-139 31 31 31 56 56 56 F F F 7 140-159 32 32 32 57 57 57 F F F 8 160-179 32 32 32 57 57 57 F F F 9 180-199 32 32 32 57 57 57 F F F 10 200-219 32 32 32 57 57 57 F F F 11 220-239 32 32 32 57 57 57 F F F 12 240-259 32 32 32 57 57 57 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 26 26 26 47 47 47 F F F 15 300-319 31 31 31 56 56 56 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 32 32 32 57 57 57 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 31 31 31 56 56 56 F F F 22 440-459 32 32 32 57 57 57 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 31 31 31 56 56 56 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 32, Max flush: 57 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.636207315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.680943609Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.681596431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.681640317Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.681662904Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.681681842Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} {"msg":"Crucible 5e044c6e-15cb-4f90-925c-7b3fc153a344 has session id: e145052a-b566-4826-95aa-03c856d9fe69","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.682297388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Upstairs opts: Upstairs UUID: 5e044c6e-15cb-4f90-925c-7b3fc153a344, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.682344443Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Crucible stats registered with UUID: 5e044c6e-15cb-4f90-925c-7b3fc153a344","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.682368901Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:51.682390088Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68285433Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.682887567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} {"msg":"5e044c6e-15cb-4f90-925c-7b3fc153a344 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68295189Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.682982017Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"0","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683030452Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"0","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683061578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"1","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683090845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"1","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683118612Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"2","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68314606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"2","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683179126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683209143Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68323767Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"ds_connection connected from Ok(127.0.0.1:37350)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683265877Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"ds_connection connected from Ok(127.0.0.1:43647)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683294804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"ds_connection connected from Ok(127.0.0.1:49460)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.683389844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.684025678Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"2","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.684053505Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"0","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.684068373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"1","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.684691148Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"0","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.684828394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"1","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68486893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"2","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685203735Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685313883Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[0]R flush_numbers[0..12]: [56, 51, 56, 57, 57, 57, 56, 57, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685433561Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[0]R generation[0..12]: [31, 28, 31, 32, 32, 32, 31, 32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685454629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685469407Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[1]R flush_numbers[0..12]: [56, 51, 56, 57, 57, 57, 56, 57, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685483226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[1]R generation[0..12]: [31, 28, 31, 32, 32, 32, 31, 32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685496744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685510683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[2]R flush_numbers[0..12]: [56, 51, 56, 57, 57, 57, 56, 57, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685524261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[2]R generation[0..12]: [31, 28, 31, 32, 32, 32, 31, 32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68553766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685551609Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685571037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Generation requested: 34 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685583885Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685596244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685608843Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"":"downstairs","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685622081Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68563696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"0","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685650348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"1","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685663377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"client":"2","":"downstairs","session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685676516Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685688444Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"5e044c6e-15cb-4f90-925c-7b3fc153a344 is now active with session: e145052a-b566-4826-95aa-03c856d9fe69","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685700923Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685712532Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328,"session_id":"e145052a-b566-4826-95aa-03c856d9fe69"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.685724481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} {"msg":"Activated sub_volume 5e044c6e-15cb-4f90-925c-7b3fc153a344","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:51.68573561Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 32768 data: 5 4 4 4 3 3 3 3 02/30 Read block 31 len 36864 03/30 Read block 231 len 4096 04/30 Read block 140 len 32768 05/30 Read block 443 len 12288 06/30 Write block 554 len 12288 data: 5 5 4 07/30 Write block 14 len 28672 data: 5 6 6 5 5 4 4 08/30 Read block 563 len 4096 09/30 Read block 238 len 16384 10/30 Write block 380 len 28672 data: 5 5 4 4 4 4 4 11/30 Write block 154 len 8192 data: 5 4 12/30 Read block 280 len 36864 13/30 Write block 36 len 24576 data: 4 4 3 3 3 3 14/30 Read block 269 len 12288 15/30 Write block 392 len 12288 data: 7 6 5 16/30 Write block 119 len 24576 data: 4 3 2 3 3 3 17/30 Write block 119 len 16384 data: 5 4 3 4 18/30 Write block 437 len 16384 data: 3 4 5 4 19/30 Read block 286 len 4096 20/30 Read block 46 len 12288 21/30 Write block 34 len 20480 data: 4 4 5 5 4 22/30 Flush 23/30 Write block 450 len 40960 data: 4 4 4 4 4 4 4 5 3 3 24/30 Flush 25/30 Write block 1 len 24576 data: 3 3 3 4 4 6 26/30 Write block 76 len 32768 data: 5 4 4 5 4 5 4 5 27/30 Write block 213 len 8192 data: 4 4 28/30 Write block 196 len 4096 data: 5 29/30 Write block 381 len 24576 data: 6 5 5 5 5 5 30/30 Read block 2 len 4096 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 6 4 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:51.705123418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 58 58 58 T T T 1 020-039 34 34 34 58 58 58 F F F 2 040-059 34 34 34 58 58 58 F F F 3 060-079 32 32 32 57 57 57 T T T 4 080-099 32 32 32 57 57 57 T T T 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 32 32 32 57 57 57 F F F 9 180-199 32 32 32 57 57 57 T T T 10 200-219 32 32 32 57 57 57 T T T 11 220-239 32 32 32 57 57 57 F F F 12 240-259 32 32 32 57 57 57 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 31 31 31 56 56 56 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 34 34 34 58 58 58 T T T 20 400-419 32 32 32 57 57 57 F F F 21 420-439 34 34 34 58 58 58 F F F 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 34, Max flush: 59 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 60 60 60 F F F 1 020-039 34 34 34 58 58 58 F F F 2 040-059 34 34 34 58 58 58 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 35 35 35 60 60 60 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 32 32 32 57 57 57 F F F 9 180-199 35 35 35 60 60 60 F F F 10 200-219 35 35 35 60 60 60 F F F 11 220-239 32 32 32 57 57 57 F F F 12 240-259 32 32 32 57 57 57 F F F 13 260-279 17 17 17 33 33 33 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 31 31 31 56 56 56 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 35 35 35 60 60 60 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 34 34 34 58 58 58 F F F 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 27 27 27 49 49 49 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 35, Max flush: 60 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:55.948174993Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:55.991154601Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:55.991561339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.022556736Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.023249324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.023293139Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.023310868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.023328906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} {"msg":"Crucible 2abb07f0-141f-4477-99e6-c5319afe995d has session id: f9a970a1-c726-492b-8f5c-1748e669828c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.023891987Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Upstairs opts: Upstairs UUID: 2abb07f0-141f-4477-99e6-c5319afe995d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.023950071Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Crucible stats registered with UUID: 2abb07f0-141f-4477-99e6-c5319afe995d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.023971269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:56.023987187Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024377986Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024406303Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} {"msg":"2abb07f0-141f-4477-99e6-c5319afe995d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024476376Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024494954Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"0","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024521981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"0","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.02453568Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"1","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024549979Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"1","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024562937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"2","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024576836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"2","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024589734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024604353Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.02462866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61326)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024654868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50325)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024896732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47719)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.024939258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.025450225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"1","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.025484341Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"2","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.025501039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"0","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.026155351Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"2","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.026296636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"1","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.0263526Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"0","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.026718812Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.02693238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[0]R flush_numbers[0..12]: [60, 58, 58, 60, 60, 58, 58, 58, 57, 60, 60, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027019731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[0]R generation[0..12]: [35, 34, 34, 35, 35, 34, 34, 34, 32, 35, 35, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027040269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027055437Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[1]R flush_numbers[0..12]: [60, 58, 58, 60, 60, 58, 58, 58, 57, 60, 60, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027069286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[1]R generation[0..12]: [35, 34, 34, 35, 35, 34, 34, 34, 32, 35, 35, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027083374Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027097653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[2]R flush_numbers[0..12]: [60, 58, 58, 60, 60, 58, 58, 58, 57, 60, 60, 57]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027111411Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[2]R generation[0..12]: [35, 34, 34, 35, 35, 34, 34, 34, 32, 35, 35, 32]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027129479Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027143938Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027156767Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027170035Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027182684Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027202232Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"":"downstairs","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.02721587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027227839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"0","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027241078Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"1","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027254146Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"client":"2","":"downstairs","session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027267215Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027284173Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"2abb07f0-141f-4477-99e6-c5319afe995d is now active with session: f9a970a1-c726-492b-8f5c-1748e669828c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027307711Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027323729Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336,"session_id":"f9a970a1-c726-492b-8f5c-1748e669828c"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027336048Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} {"msg":"Activated sub_volume 2abb07f0-141f-4477-99e6-c5319afe995d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:00:56.027347247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 313 len 4096 data: 3 02/30 Write block 176 len 8192 data: 8 5 03/30 Read block 283 len 20480 04/30 Write block 222 len 40960 data: 4 3 3 3 3 3 4 4 5 5 05/30 Read block 113 len 24576 06/30 Read block 348 len 32768 07/30 Read block 154 len 36864 08/30 Read block 146 len 20480 09/30 Read block 24 len 4096 10/30 Write block 176 len 16384 data: 9 6 5 5 11/30 Read block 587 len 32768 12/30 Write block 270 len 20480 data: 4 4 3 3 3 13/30 Write block 282 len 12288 data: 5 6 6 14/30 Read block 311 len 24576 15/30 Read block 272 len 24576 16/30 Write block 92 len 8192 data: 2 2 17/30 Read block 212 len 12288 18/30 Write block 82 len 32768 data: 5 6 5 4 3 4 3 2 19/30 Read block 385 len 24576 20/30 Write block 485 len 4096 data: 5 21/30 Write block 51 len 8192 data: 4 4 22/30 Flush 23/30 Write block 8 len 28672 data: 5 6 6 7 7 5 6 24/30 Read block 430 len 12288 25/30 Write block 225 len 32768 data: 4 4 4 5 5 6 6 4 26/30 Read block 297 len 24576 27/30 Write block 382 len 12288 data: 6 6 6 28/30 Write block 197 len 20480 data: 3 3 4 4 4 29/30 Read block 589 len 4096 30/30 Read block 579 len 28672 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 7 8 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:00:56.046816446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 60 60 60 T T T 1 020-039 34 34 34 58 58 58 F F F 2 040-059 36 36 36 61 61 61 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 36 36 36 61 61 61 F F F 9 180-199 35 35 35 60 60 60 T T T 10 200-219 35 35 35 60 60 60 T T T 11 220-239 36 36 36 61 61 61 T T T 12 240-259 32 32 32 57 57 57 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 36 36 36 61 61 61 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 35 35 35 60 60 60 T T T 20 400-419 32 32 32 57 57 57 F F F 21 420-439 34 34 34 58 58 58 F F F 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 36, Max flush: 61 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 F F F 1 020-039 34 34 34 58 58 58 F F F 2 040-059 36 36 36 61 61 61 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 36 36 36 61 61 61 F F F 9 180-199 37 37 37 62 62 62 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 32 32 32 57 57 57 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 36 36 36 61 61 61 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 32 32 32 57 57 57 F F F 18 360-379 31 31 31 56 56 56 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 34 34 34 58 58 58 F F F 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 37, Max flush: 62 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.317790739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.362311066Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.3630426Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.363079166Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.363093435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.363107903Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} {"msg":"Crucible a8960ddc-9077-4b2b-bb47-3fbd61a1641b has session id: e3433f82-bf23-4cb7-a49f-a50bdf8faa90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.363754636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Upstairs opts: Upstairs UUID: a8960ddc-9077-4b2b-bb47-3fbd61a1641b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.363792862Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Crucible stats registered with UUID: a8960ddc-9077-4b2b-bb47-3fbd61a1641b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.36380898Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:00.363821739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364256754Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364281591Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} {"msg":"a8960ddc-9077-4b2b-bb47-3fbd61a1641b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364371422Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.36438948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"0","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364429176Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"0","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364456653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"1","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.36448506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"1","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364513067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"2","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364540944Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"2","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364569321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364597138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364630855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"ds_connection connected from Ok(127.0.0.1:43367)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364804346Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"ds_connection connected from Ok(127.0.0.1:58313)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364844912Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"ds_connection connected from Ok(127.0.0.1:62538)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.364871719Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.36544069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"1","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.365514203Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"0","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.365622541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"2","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366145037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"1","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366185782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"0","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366216579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"2","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366723017Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366849963Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[0]R flush_numbers[0..12]: [62, 58, 61, 60, 61, 58, 58, 58, 61, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366890599Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[0]R generation[0..12]: [37, 34, 36, 35, 36, 34, 34, 34, 36, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366907367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366921856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[1]R flush_numbers[0..12]: [62, 58, 61, 60, 61, 58, 58, 58, 61, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366939414Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[1]R generation[0..12]: [37, 34, 36, 35, 36, 34, 34, 34, 36, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366953582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366967521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[2]R flush_numbers[0..12]: [62, 58, 61, 60, 61, 58, 58, 58, 61, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366981119Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[2]R generation[0..12]: [37, 34, 36, 35, 36, 34, 34, 34, 36, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.366994468Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367008197Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367021305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367034364Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367047203Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367059981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"":"downstairs","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.3670737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367086008Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367107586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"1","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367130954Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"client":"2","":"downstairs","session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367155721Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.36717005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"a8960ddc-9077-4b2b-bb47-3fbd61a1641b is now active with session: e3433f82-bf23-4cb7-a49f-a50bdf8faa90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367188628Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367210775Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344,"session_id":"e3433f82-bf23-4cb7-a49f-a50bdf8faa90"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367225004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} {"msg":"Activated sub_volume a8960ddc-9077-4b2b-bb47-3fbd61a1641b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:00.367236293Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 16384 data: 3 2 2 3 02/30 Write block 176 len 8192 data: 10 7 03/30 Read block 91 len 24576 04/30 Read block 146 len 12288 05/30 Read block 359 len 20480 06/30 Write block 341 len 12288 data: 3 4 3 07/30 Write block 284 len 36864 data: 7 4 3 5 5 5 6 5 5 08/30 Read block 112 len 16384 09/30 Read block 192 len 12288 10/30 Write block 284 len 16384 data: 8 5 4 6 11/30 Flush 12/30 Flush 13/30 Write block 425 len 4096 data: 4 14/30 Read block 208 len 16384 15/30 Read block 544 len 40960 16/30 Read block 319 len 16384 17/30 Read block 188 len 16384 18/30 Read block 194 len 36864 19/30 Read block 125 len 20480 20/30 Read block 444 len 16384 21/30 Write block 153 len 12288 data: 3 6 5 22/30 Write block 152 len 8192 data: 3 4 23/30 Read block 400 len 32768 24/30 Read block 316 len 40960 25/30 Read block 2 len 12288 26/30 Read block 392 len 4096 27/30 Write block 176 len 8192 data: 11 8 28/30 Read block 96 len 20480 29/30 Read block 277 len 20480 30/30 Read block 247 len 8192 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 1 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:00.38711315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 F F F 1 020-039 34 34 34 58 58 58 F F F 2 040-059 36 36 36 61 61 61 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 34 34 34 58 58 58 T T T 8 160-179 38 38 38 63 63 63 T T T 9 180-199 37 37 37 62 62 62 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 32 32 32 57 57 57 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 38 38 38 63 63 63 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 38 38 38 63 63 63 F F F 18 360-379 38 38 38 63 63 63 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 34 34 34 58 58 58 T T T 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 38, Max flush: 63 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 F F F 1 020-039 34 34 34 58 58 58 F F F 2 040-059 36 36 36 61 61 61 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 34 34 34 58 58 58 F F F 7 140-159 39 39 39 64 64 64 F F F 8 160-179 39 39 39 64 64 64 F F F 9 180-199 37 37 37 62 62 62 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 32 32 32 57 57 57 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 38 38 38 63 63 63 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 38 38 38 63 63 63 F F F 18 360-379 38 38 38 63 63 63 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 39 39 39 64 64 64 F F F 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 39, Max flush: 64 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.615918201Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.659984266Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.660652866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.660688622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.660702861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.660717259Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} {"msg":"Crucible 155bacf4-06ea-4f8e-929b-43c1b8514460 has session id: b230a623-4786-4226-acbe-5dd5adbf8147","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.66128806Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Upstairs opts: Upstairs UUID: 155bacf4-06ea-4f8e-929b-43c1b8514460, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661322766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Crucible stats registered with UUID: 155bacf4-06ea-4f8e-929b-43c1b8514460","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661337724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:04.661350663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661797307Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661832013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} {"msg":"155bacf4-06ea-4f8e-929b-43c1b8514460 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661887037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661914964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"0","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661946051Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"0","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661965569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"1","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.661980477Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"1","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662010384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"2","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662029112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"2","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.66205403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662080607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662106304Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"ds_connection connected from Ok(127.0.0.1:39209)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662134101Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"ds_connection connected from Ok(127.0.0.1:54079)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662159619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"ds_connection connected from Ok(127.0.0.1:43667)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662184636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.66282192Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"0","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.662862796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"2","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.663388061Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"1","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.663531586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"0","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.663572522Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"2","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.663913286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"1","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664155341Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664243781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[0]R flush_numbers[0..12]: [62, 58, 61, 60, 61, 58, 58, 64, 64, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664426013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[0]R generation[0..12]: [37, 34, 36, 35, 36, 34, 34, 39, 39, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.66444573Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664460859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[1]R flush_numbers[0..12]: [62, 58, 61, 60, 61, 58, 58, 64, 64, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664474907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[1]R generation[0..12]: [37, 34, 36, 35, 36, 34, 34, 39, 39, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664488556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664502534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[2]R flush_numbers[0..12]: [62, 58, 61, 60, 61, 58, 58, 64, 64, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664515993Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[2]R generation[0..12]: [37, 34, 36, 35, 36, 34, 34, 39, 39, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664529462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.66454321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664555899Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664572477Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:04.664586536Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664603074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"":"downstairs","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664616173Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664628001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"0","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.66464121Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"1","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664654029Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"client":"2","":"downstairs","session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664667117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664679196Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"155bacf4-06ea-4f8e-929b-43c1b8514460 is now active with session: b230a623-4786-4226-acbe-5dd5adbf8147","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664691885Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664703724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352,"session_id":"b230a623-4786-4226-acbe-5dd5adbf8147"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664715712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} {"msg":"Activated sub_volume 155bacf4-06ea-4f8e-929b-43c1b8514460","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:04.664726911Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 380 len 20480 data: 6 7 7 7 7 03/30 Read block 140 len 28672 04/30 Read block 578 len 12288 05/30 Write block 124 len 8192 data: 4 5 06/30 Write block 32 len 8192 data: 4 4 07/30 Read block 535 len 32768 08/30 Read block 536 len 8192 09/30 Read block 463 len 36864 10/30 Read block 136 len 28672 11/30 Flush 12/30 Write block 355 len 28672 data: 6 5 5 4 4 4 4 13/30 Read block 596 len 16384 14/30 Write block 370 len 4096 data: 3 15/30 Write block 283 len 40960 data: 7 9 6 5 7 6 6 7 6 6 16/30 Read block 450 len 4096 17/30 Write block 544 len 24576 data: 3 4 4 4 4 3 18/30 Read block 80 len 32768 19/30 Write block 254 len 4096 data: 4 20/30 Read block 36 len 4096 21/30 Read block 455 len 12288 22/30 Flush 23/30 Write block 13 len 16384 data: 6 7 7 7 24/30 Write block 274 len 32768 data: 4 3 4 4 3 3 4 4 25/30 Read block 33 len 8192 26/30 Read block 208 len 16384 27/30 Write block 38 len 28672 data: 5 4 4 4 3 4 4 28/30 Read block 583 len 4096 29/30 Read block 266 len 28672 30/30 Read block 32 len 20480 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:04.685581956Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 37 37 37 62 62 62 T T T 1 020-039 40 40 40 66 66 66 T T T 2 040-059 36 36 36 61 61 61 T T T 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 39 39 39 64 64 64 F F F 8 160-179 39 39 39 64 64 64 F F F 9 180-199 37 37 37 62 62 62 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 36 36 36 61 61 61 T T T 14 280-299 40 40 40 67 67 67 T T T 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 40 40 40 67 67 67 F F F 18 360-379 40 40 40 67 67 67 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 39 39 39 64 64 64 F F F 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 40, Max flush: 67 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 41 41 41 68 68 68 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 39 39 39 64 64 64 F F F 8 160-179 39 39 39 64 64 64 F F F 9 180-199 37 37 37 62 62 62 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 41 41 41 68 68 68 F F F 14 280-299 41 41 41 68 68 68 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 40 40 40 67 67 67 F F F 18 360-379 40 40 40 67 67 67 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 39 39 39 64 64 64 F F F 22 440-459 34 34 34 59 59 59 F F F 23 460-479 29 29 29 54 54 54 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 30 30 30 55 55 55 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 27 27 27 49 49 49 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.924072356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:08.969152844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:08.969528725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.001825267Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.002733153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.002769829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.002783707Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.002797966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} {"msg":"Crucible d08ba018-e943-4482-9eeb-e5fb2b61ca2c has session id: 257c8c39-aebf-40ed-b929-bfcf3411fa3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.003422861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Upstairs opts: Upstairs UUID: d08ba018-e943-4482-9eeb-e5fb2b61ca2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.003465106Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Crucible stats registered with UUID: d08ba018-e943-4482-9eeb-e5fb2b61ca2c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.003486394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:09.003500393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.003953975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.003979343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} {"msg":"d08ba018-e943-4482-9eeb-e5fb2b61ca2c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004039606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004066403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"0","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004106639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"0","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004135246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"1","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004164533Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"1","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.00419081Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"2","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004219108Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"2","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004244855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004271612Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004297819Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64128)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004322847Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33115)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004356693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52673)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.004383581Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.005135512Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"1","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.005168699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"2","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.005301325Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"0","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.005803902Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"1","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.005834229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"2","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.005851857Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"0","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006407159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006517308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[0]R flush_numbers[0..12]: [68, 68, 68, 60, 61, 58, 66, 64, 64, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006589291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[0]R generation[0..12]: [41, 41, 41, 35, 36, 34, 40, 39, 39, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006609138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006624797Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[1]R flush_numbers[0..12]: [68, 68, 68, 60, 61, 58, 66, 64, 64, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006639305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[1]R generation[0..12]: [41, 41, 41, 35, 36, 34, 40, 39, 39, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006657393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006671802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[2]R flush_numbers[0..12]: [68, 68, 68, 60, 61, 58, 66, 64, 64, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.00668523Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[2]R generation[0..12]: [41, 41, 41, 35, 36, 34, 40, 39, 39, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006698969Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006712867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006725906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006738905Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006751634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006764482Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"":"downstairs","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006777621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:09.006789619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"0","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006806888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"1","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006819936Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"client":"2","":"downstairs","session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006833055Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006845284Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"d08ba018-e943-4482-9eeb-e5fb2b61ca2c is now active with session: 257c8c39-aebf-40ed-b929-bfcf3411fa3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006859972Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006872131Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360,"session_id":"257c8c39-aebf-40ed-b929-bfcf3411fa3e"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.00688418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} {"msg":"Activated sub_volume d08ba018-e943-4482-9eeb-e5fb2b61ca2c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:09.006895299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 452 len 40960 data: 5 5 5 5 5 6 4 4 4 4 02/30 Write block 277 len 40960 data: 5 4 4 5 5 6 8 10 7 6 03/30 Read block 167 len 16384 04/30 Read block 148 len 8192 05/30 Read block 438 len 32768 06/30 Read block 404 len 16384 07/30 Read block 44 len 20480 08/30 Flush 09/30 Write block 504 len 36864 data: 5 3 3 3 2 4 5 5 4 10/30 Flush 11/30 Write block 294 len 16384 data: 4 4 4 4 12/30 Read block 23 len 16384 13/30 Write block 567 len 4096 data: 3 14/30 Read block 192 len 28672 15/30 Read block 510 len 12288 16/30 Flush 17/30 Write block 570 len 24576 data: 4 4 5 4 5 5 18/30 Read block 288 len 16384 19/30 Flush 20/30 Write block 188 len 32768 data: 4 4 4 5 4 4 4 4 21/30 Read block 217 len 16384 22/30 Read block 322 len 36864 23/30 Flush 24/30 Write block 29 len 24576 data: 3 4 4 5 5 5 25/30 Read block 79 len 32768 26/30 Read block 540 len 8192 27/30 Write block 115 len 32768 data: 2 2 3 3 6 5 4 5 28/30 Read block 51 len 12288 29/30 Read block 544 len 32768 30/30 Read block 110 len 20480 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:09.02874585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 41 41 41 68 68 68 T T T 2 040-059 41 41 41 68 68 68 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 34 34 34 58 58 58 T T T 6 120-139 40 40 40 66 66 66 T T T 7 140-159 39 39 39 64 64 64 F F F 8 160-179 39 39 39 64 64 64 F F F 9 180-199 42 42 42 73 73 73 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 42 42 42 71 71 71 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 40 40 40 67 67 67 F F F 18 360-379 40 40 40 67 67 67 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 39 39 39 64 64 64 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 42 42 42 72 72 72 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 42, Max flush: 73 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 43 43 43 74 74 74 F F F 2 040-059 41 41 41 68 68 68 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 43 43 43 74 74 74 F F F 6 120-139 43 43 43 74 74 74 F F F 7 140-159 39 39 39 64 64 64 F F F 8 160-179 39 39 39 64 64 64 F F F 9 180-199 42 42 42 73 73 73 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 42 42 42 71 71 71 F F F 15 300-319 36 36 36 61 61 61 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 40 40 40 67 67 67 F F F 18 360-379 40 40 40 67 67 67 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 32 32 32 57 57 57 F F F 21 420-439 39 39 39 64 64 64 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 36 36 36 61 61 61 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 42 42 42 72 72 72 F F F 29 580-599 24 24 24 45 45 45 F F F Max gen: 43, Max flush: 74 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.283794135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:13.326843716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.327236245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.358868946Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.359454415Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.35950028Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.359517929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.359536987Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} {"msg":"Crucible 39c86f75-532d-44de-b86d-b438fdaec95d has session id: e302fbf8-e88f-460a-bf2f-58dec4dd9222","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360055893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Upstairs opts: Upstairs UUID: 39c86f75-532d-44de-b86d-b438fdaec95d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360106597Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Crucible stats registered with UUID: 39c86f75-532d-44de-b86d-b438fdaec95d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360151573Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:13.36017484Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360609145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360633552Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} {"msg":"39c86f75-532d-44de-b86d-b438fdaec95d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360689436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360713814Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"0","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360742521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"0","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360758089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"1","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360772798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"1","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360792416Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"2","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360819033Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"2","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.36084246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360866608Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360889716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"ds_connection connected from Ok(127.0.0.1:65056)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360915573Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"ds_connection connected from Ok(127.0.0.1:50661)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.36094128Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"ds_connection connected from Ok(127.0.0.1:44111)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.360965518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.361744837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"0","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.361785182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"1","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.36180172Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"2","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.362393719Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"2","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.362512177Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"1","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.362556072Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"0","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.362904785Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363101505Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[0]R flush_numbers[0..12]: [68, 74, 68, 60, 61, 74, 74, 64, 64, 73, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363219043Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[0]R generation[0..12]: [41, 43, 41, 35, 36, 43, 43, 39, 39, 42, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363239211Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363254379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[1]R flush_numbers[0..12]: [68, 74, 68, 60, 61, 74, 74, 64, 64, 73, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363268698Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[1]R generation[0..12]: [41, 43, 41, 35, 36, 43, 43, 39, 39, 42, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363283066Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363297055Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[2]R flush_numbers[0..12]: [68, 74, 68, 60, 61, 74, 74, 64, 64, 73, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363310933Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[2]R generation[0..12]: [41, 43, 41, 35, 36, 43, 43, 39, 39, 42, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363324712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.36333869Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363351629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363364577Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:13.363377306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363398094Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"":"downstairs","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363411783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363423682Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"0","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.36343664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"1","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363449459Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"client":"2","":"downstairs","session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363462407Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363475366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"39c86f75-532d-44de-b86d-b438fdaec95d is now active with session: e302fbf8-e88f-460a-bf2f-58dec4dd9222","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363488165Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363500014Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368,"session_id":"e302fbf8-e88f-460a-bf2f-58dec4dd9222"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363511992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} {"msg":"Activated sub_volume 39c86f75-532d-44de-b86d-b438fdaec95d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:13.363523171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 16384 data: 5 4 5 5 02/30 Write block 144 len 8192 data: 3 3 03/30 Read block 97 len 24576 04/30 Read block 208 len 36864 05/30 Write block 413 len 32768 data: 3 3 3 3 5 6 6 3 06/30 Read block 426 len 4096 07/30 Read block 386 len 40960 08/30 Read block 532 len 36864 09/30 Write block 194 len 16384 data: 5 5 6 4 10/30 Write block 156 len 36864 data: 5 5 6 6 6 4 3 4 4 11/30 Read block 492 len 8192 12/30 Write block 487 len 16384 data: 5 5 5 5 13/30 Write block 361 len 28672 data: 5 4 3 3 3 2 2 14/30 Read block 3 len 4096 15/30 Read block 250 len 20480 16/30 Read block 480 len 8192 17/30 Write block 411 len 4096 data: 3 18/30 Write block 302 len 28672 data: 5 5 6 6 4 4 4 19/30 Write block 290 len 8192 data: 8 7 20/30 Read block 250 len 28672 21/30 Read block 412 len 28672 22/30 Write block 187 len 32768 data: 4 5 5 5 6 5 5 6 23/30 Read block 254 len 20480 24/30 Write block 152 len 8192 data: 4 5 25/30 Write block 429 len 32768 data: 6 5 7 8 7 8 5 4 26/30 Write block 115 len 36864 data: 3 3 4 4 7 6 5 6 4 27/30 Write block 589 len 4096 data: 4 28/30 Read block 27 len 16384 29/30 Write block 585 len 4096 data: 5 30/30 Read block 106 len 40960 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 30 29 28 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:13.382599242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 43 43 43 74 74 74 F F F 2 040-059 41 41 41 68 68 68 T T T 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 43 43 43 74 74 74 T T T 6 120-139 43 43 43 74 74 74 T T T 7 140-159 39 39 39 64 64 64 T T T 8 160-179 39 39 39 64 64 64 T T T 9 180-199 42 42 42 73 73 73 T T T 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 42 42 42 71 71 71 T T T 15 300-319 36 36 36 61 61 61 T T T 16 320-339 32 32 32 57 57 57 F F F 17 340-359 40 40 40 67 67 67 F F F 18 360-379 40 40 40 67 67 67 T T T 19 380-399 40 40 40 66 66 66 F F F 20 400-419 32 32 32 57 57 57 T T T 21 420-439 39 39 39 64 64 64 T T T 22 440-459 42 42 42 69 69 69 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 36 36 36 61 61 61 T T T 25 500-519 42 42 42 70 70 70 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 42 42 42 72 72 72 F F F 29 580-599 24 24 24 45 45 45 T T T Max gen: 43, Max flush: 74 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 43 43 43 74 74 74 F F F 2 040-059 44 44 44 75 75 75 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 44 44 44 75 75 75 F F F 6 120-139 44 44 44 75 75 75 F F F 7 140-159 44 44 44 75 75 75 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 44 44 44 75 75 75 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 40 40 40 67 67 67 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 44 44 44 75 75 75 F F F 15 300-319 44 44 44 75 75 75 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 40 40 40 67 67 67 F F F 18 360-379 44 44 44 75 75 75 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 44 44 44 75 75 75 F F F 21 420-439 44 44 44 75 75 75 F F F 22 440-459 42 42 42 69 69 69 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 44 44 44 75 75 75 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 40 40 40 67 67 67 F F F 28 560-579 42 42 42 72 72 72 F F F 29 580-599 44 44 44 75 75 75 F F F Max gen: 44, Max flush: 75 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.693631038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.739819421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.740476383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.740512849Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.740527607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.740541836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} {"msg":"Crucible ba81cf7f-afcd-4879-ac4f-2cd262a723c0 has session id: 2df780a9-693f-4945-9592-91ef5e791ebe","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741193228Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Upstairs opts: Upstairs UUID: ba81cf7f-afcd-4879-ac4f-2cd262a723c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741242283Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Crucible stats registered with UUID: ba81cf7f-afcd-4879-ac4f-2cd262a723c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.74126798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:17.741288928Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741691596Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741719663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} {"msg":"ba81cf7f-afcd-4879-ac4f-2cd262a723c0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741771548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741798125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"0","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741828412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"0","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.74184271Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"1","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741856789Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"1","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741869738Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"2","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741884146Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"2","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741897365Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741911763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.741925482Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"ds_connection connected from Ok(127.0.0.1:37076)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.742082285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"ds_connection connected from Ok(127.0.0.1:39841)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.742264037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"ds_connection connected from Ok(127.0.0.1:34136)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.742299983Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.742838326Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"1","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.742867794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"0","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.743600637Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"1","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.743640683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"0","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.744346209Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.744457448Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.744489184Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"2","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745026898Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"2","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[0]R flush_numbers[0..12]: [68, 74, 75, 60, 61, 75, 75, 75, 75, 75, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745535275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[0]R generation[0..12]: [41, 43, 44, 35, 36, 44, 44, 44, 44, 44, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745556803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745572201Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[1]R flush_numbers[0..12]: [68, 74, 75, 60, 61, 75, 75, 75, 75, 75, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.74559009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[1]R generation[0..12]: [41, 43, 44, 35, 36, 44, 44, 44, 44, 44, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745604308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745618237Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[2]R flush_numbers[0..12]: [68, 74, 75, 60, 61, 75, 75, 75, 75, 75, 62, 62]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745631955Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[2]R generation[0..12]: [41, 43, 44, 35, 36, 44, 44, 44, 44, 44, 37, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745645464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745659342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Max found gen is 45","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745672231Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Generation requested: 46 >= found:45","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.74568523Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:17.745698258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745716326Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"":"downstairs","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745729675Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745741624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"0","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745754992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"1","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745768381Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"client":"2","":"downstairs","session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745781879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745798458Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"ba81cf7f-afcd-4879-ac4f-2cd262a723c0 is now active with session: 2df780a9-693f-4945-9592-91ef5e791ebe","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745811567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745823565Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376,"session_id":"2df780a9-693f-4945-9592-91ef5e791ebe"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745835664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} {"msg":"Activated sub_volume ba81cf7f-afcd-4879-ac4f-2cd262a723c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:17.745846633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 224 len 8192 data: 4 5 02/30 Write block 502 len 12288 data: 6 5 6 03/30 Read block 426 len 24576 04/30 Read block 302 len 16384 05/30 Read block 41 len 28672 06/30 Write block 445 len 20480 data: 4 4 4 2 4 07/30 Read block 429 len 20480 08/30 Read block 138 len 20480 09/30 Write block 342 len 28672 data: 5 4 3 3 3 3 3 10/30 Read block 358 len 16384 11/30 Read block 434 len 8192 12/30 Write block 541 len 8192 data: 4 4 13/30 Write block 350 len 36864 data: 3 4 5 6 6 7 6 6 5 14/30 Read block 345 len 40960 15/30 Flush 16/30 Write block 357 len 28672 data: 7 6 5 5 6 5 4 17/30 Write block 286 len 36864 data: 7 8 7 7 9 8 7 5 5 18/30 Read block 472 len 20480 19/30 Read block 250 len 4096 20/30 Read block 179 len 20480 21/30 Read block 81 len 20480 22/30 Flush 23/30 Write block 289 len 16384 data: 8 10 9 8 24/30 Write block 222 len 4096 data: 5 25/30 Read block 164 len 36864 26/30 Flush 27/30 Write block 236 len 32768 data: 4 5 4 4 3 7 7 7 28/30 Write block 560 len 36864 data: 3 3 2 2 2 3 3 4 3 29/30 Read block 414 len 12288 30/30 Write block 239 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:17.766055785Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 43 43 43 74 74 74 F F F 2 040-059 44 44 44 75 75 75 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 44 44 44 75 75 75 F F F 6 120-139 44 44 44 75 75 75 F F F 7 140-159 44 44 44 75 75 75 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 44 44 44 75 75 75 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 46 46 46 78 78 78 T T T 12 240-259 40 40 40 67 67 67 T T T 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 78 78 78 F F F 15 300-319 44 44 44 75 75 75 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 46 46 46 77 77 77 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 44 44 44 75 75 75 F F F 21 420-439 44 44 44 75 75 75 F F F 22 440-459 46 46 46 76 76 76 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 44 44 44 75 75 75 F F F 25 500-519 46 46 46 76 76 76 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 46 46 46 76 76 76 F F F 28 560-579 42 42 42 72 72 72 T T T 29 580-599 44 44 44 75 75 75 F F F Max gen: 46, Max flush: 78 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 F F F 1 020-039 43 43 43 74 74 74 F F F 2 040-059 44 44 44 75 75 75 F F F 3 060-079 35 35 35 60 60 60 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 44 44 44 75 75 75 F F F 6 120-139 44 44 44 75 75 75 F F F 7 140-159 44 44 44 75 75 75 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 44 44 44 75 75 75 F F F 10 200-219 37 37 37 62 62 62 F F F 11 220-239 47 47 47 79 79 79 F F F 12 240-259 47 47 47 79 79 79 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 78 78 78 F F F 15 300-319 44 44 44 75 75 75 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 46 46 46 77 77 77 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 44 44 44 75 75 75 F F F 21 420-439 44 44 44 75 75 75 F F F 22 440-459 46 46 46 76 76 76 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 44 44 44 75 75 75 F F F 25 500-519 46 46 46 76 76 76 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 46 46 46 76 76 76 F F F 28 560-579 47 47 47 79 79 79 F F F 29 580-599 44 44 44 75 75 75 F F F Max gen: 47, Max flush: 79 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:21.995736904Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.040987085Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.04180342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.041841016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.041860674Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.041876832Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} {"msg":"Crucible 7e51444c-812b-4ad0-82f8-0c2c01df6c83 has session id: b52d5cb2-480e-43d0-8c7b-698fc967e3a2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.042382129Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Upstairs opts: Upstairs UUID: 7e51444c-812b-4ad0-82f8-0c2c01df6c83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.042499167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Crucible stats registered with UUID: 7e51444c-812b-4ad0-82f8-0c2c01df6c83","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.042536223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:22.042561441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.042977447Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043003055Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} {"msg":"7e51444c-812b-4ad0-82f8-0c2c01df6c83 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043096555Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043114683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"0","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.04314388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"0","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043158588Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"1","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043173027Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"1","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043190675Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"2","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043206033Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"2","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043219482Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.04323444Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043272996Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34518)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043357518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53063)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043394394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:46876)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.043419441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.04419329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"0","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.044222107Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"1","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.044266453Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"2","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.044971879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"1","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.044999736Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"0","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045067709Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"2","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045627701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045658258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[0]R flush_numbers[0..12]: [68, 74, 75, 60, 61, 75, 75, 75, 75, 75, 62, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045679145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[0]R generation[0..12]: [41, 43, 44, 35, 36, 44, 44, 44, 44, 44, 37, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045694124Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045708722Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[1]R flush_numbers[0..12]: [68, 74, 75, 60, 61, 75, 75, 75, 75, 75, 62, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045722651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[1]R generation[0..12]: [41, 43, 44, 35, 36, 44, 44, 44, 44, 44, 37, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045736399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045750918Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[2]R flush_numbers[0..12]: [68, 74, 75, 60, 61, 75, 75, 75, 75, 75, 62, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045764856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[2]R generation[0..12]: [41, 43, 44, 35, 36, 44, 44, 44, 44, 44, 37, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045778255Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045793024Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045812762Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045832739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045854547Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045878225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"":"downstairs","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045905582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.04591954Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"0","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045933279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"1","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045956096Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"client":"2","":"downstairs","session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.045988063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.046011831Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"7e51444c-812b-4ad0-82f8-0c2c01df6c83 is now active with session: b52d5cb2-480e-43d0-8c7b-698fc967e3a2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.046035508Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.046049377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384,"session_id":"b52d5cb2-480e-43d0-8c7b-698fc967e3a2"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.046061585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} {"msg":"Activated sub_volume 7e51444c-812b-4ad0-82f8-0c2c01df6c83","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:22.046075444Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 422 len 20480 data: 2 2 3 5 4 02/30 Read block 115 len 32768 03/30 Flush 04/30 Write block 42 len 12288 data: 5 6 6 05/30 Flush 06/30 Write block 569 len 4096 data: 3 07/30 Write block 438 len 40960 data: 5 6 5 3 2 3 3 5 5 5 08/30 Write block 58 len 8192 data: 7 7 09/30 Read block 524 len 16384 10/30 Write block 209 len 28672 data: 4 4 4 3 5 5 3 11/30 Read block 577 len 4096 12/30 Read block 453 len 20480 13/30 Write block 58 len 12288 data: 8 8 7 14/30 Write block 127 len 8192 data: 5 5 15/30 Read block 460 len 12288 16/30 Write block 9 len 16384 data: 7 7 8 8 17/30 Read block 287 len 28672 18/30 Write block 184 len 40960 data: 4 3 3 5 6 6 6 7 6 6 19/30 Read block 404 len 8192 20/30 Read block 187 len 28672 21/30 Write block 417 len 20480 data: 6 7 7 4 3 22/30 Write block 440 len 24576 data: 6 4 3 4 4 6 23/30 Read block 439 len 12288 24/30 Write block 307 len 28672 data: 5 5 4 5 5 4 4 25/30 Read block 582 len 32768 26/30 Read block 560 len 4096 27/30 Write block 124 len 24576 data: 5 6 5 6 6 4 28/30 Read block 115 len 8192 29/30 Read block 322 len 40960 30/30 Write block 298 len 12288 data: 4 5 5 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 1 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:22.066207115Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 68 68 68 T T T 1 020-039 43 43 43 74 74 74 F F F 2 040-059 48 48 48 81 81 81 T T T 3 060-079 35 35 35 60 60 60 T T T 4 080-099 36 36 36 61 61 61 F F F 5 100-119 44 44 44 75 75 75 F F F 6 120-139 44 44 44 75 75 75 T T T 7 140-159 44 44 44 75 75 75 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 44 44 44 75 75 75 T T T 10 200-219 37 37 37 62 62 62 T T T 11 220-239 47 47 47 79 79 79 F F F 12 240-259 47 47 47 79 79 79 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 78 78 78 T T T 15 300-319 44 44 44 75 75 75 T T T 16 320-339 32 32 32 57 57 57 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 46 46 46 77 77 77 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 44 44 44 75 75 75 T T T 21 420-439 48 48 48 80 80 80 T T T 22 440-459 46 46 46 76 76 76 T T T 23 460-479 42 42 42 69 69 69 F F F 24 480-499 44 44 44 75 75 75 F F F 25 500-519 46 46 46 76 76 76 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 46 46 46 76 76 76 F F F 28 560-579 47 47 47 79 79 79 T T T 29 580-599 44 44 44 75 75 75 F F F Max gen: 48, Max flush: 81 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 82 82 82 F F F 1 020-039 43 43 43 74 74 74 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 49 49 49 82 82 82 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 44 44 44 75 75 75 F F F 6 120-139 49 49 49 82 82 82 F F F 7 140-159 44 44 44 75 75 75 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 49 49 49 82 82 82 F F F 10 200-219 49 49 49 82 82 82 F F F 11 220-239 47 47 47 79 79 79 F F F 12 240-259 47 47 47 79 79 79 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 49 49 49 82 82 82 F F F 15 300-319 49 49 49 82 82 82 F F F 16 320-339 32 32 32 57 57 57 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 46 46 46 77 77 77 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 49 49 49 82 82 82 F F F 22 440-459 49 49 49 82 82 82 F F F 23 460-479 42 42 42 69 69 69 F F F 24 480-499 44 44 44 75 75 75 F F F 25 500-519 46 46 46 76 76 76 F F F 26 520-539 32 32 32 57 57 57 F F F 27 540-559 46 46 46 76 76 76 F F F 28 560-579 49 49 49 82 82 82 F F F 29 580-599 44 44 44 75 75 75 F F F Max gen: 49, Max flush: 82 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.341782269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.384748359Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.385156886Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.415838806Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.416454342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.416489449Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.416503127Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.416517456Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} {"msg":"Crucible 0276b4c5-7003-4f24-8b7b-c25c88332be5 has session id: f166ac8b-75fd-4ab2-826e-35910fcf109e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417082997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Upstairs opts: Upstairs UUID: 0276b4c5-7003-4f24-8b7b-c25c88332be5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417126982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Crucible stats registered with UUID: 0276b4c5-7003-4f24-8b7b-c25c88332be5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.41714908Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.417167328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417548088Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417572146Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} {"msg":"0276b4c5-7003-4f24-8b7b-c25c88332be5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.41762267Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417649857Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"0","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417681834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"0","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417697183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"1","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417711651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"1","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.41772489Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"2","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417738818Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"2","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417751907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417773705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417799582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42794)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.417830949Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58410)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.418004071Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41055)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.418043856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.418572161Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"1","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.418611007Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"2","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.418787539Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"0","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.419301995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"2","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.419342731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"1","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.419414573Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"0","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.419959997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.419994793Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[0]R flush_numbers[0..12]: [82, 74, 82, 82, 61, 75, 82, 75, 75, 82, 82, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420054037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[0]R generation[0..12]: [49, 43, 49, 49, 36, 44, 49, 44, 44, 49, 49, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420083204Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420099632Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[1]R flush_numbers[0..12]: [82, 74, 82, 82, 61, 75, 82, 75, 75, 82, 82, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.42011739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[1]R generation[0..12]: [49, 43, 49, 49, 36, 44, 49, 44, 44, 49, 49, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420132529Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420146587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[2]R flush_numbers[0..12]: [82, 74, 82, 82, 61, 75, 82, 75, 75, 82, 82, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420160376Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[2]R generation[0..12]: [49, 43, 49, 49, 36, 44, 49, 44, 44, 49, 49, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420173864Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420187973Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420200711Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.42021378Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420226409Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420239107Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"":"downstairs","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:01:26.420252286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420271074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"0","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420284373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"1","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420297272Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"client":"2","":"downstairs","session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.42031019Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420324309Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"0276b4c5-7003-4f24-8b7b-c25c88332be5 is now active with session: f166ac8b-75fd-4ab2-826e-35910fcf109e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420337257Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420349116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420361145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} {"msg":"Activated sub_volume 0276b4c5-7003-4f24-8b7b-c25c88332be5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:26.420372214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 36864 data: 5 4 3 3 4 6 5 5 6 02/30 Read block 488 len 40960 03/30 Write block 17 len 16384 data: 6 6 5 5 04/30 Flush 05/30 Write block 563 len 20480 data: 3 3 4 4 5 06/30 Read block 310 len 16384 07/30 Read block 15 len 24576 08/30 Flush 09/30 Write block 61 len 16384 data: 6 5 5 4 10/30 Write block 291 len 36864 data: 10 9 6 6 5 5 5 5 6 11/30 Write block 471 len 20480 data: 3 3 3 3 4 12/30 Write block 528 len 24576 data: 5 5 4 4 3 2 13/30 Flush 14/30 Write block 141 len 12288 data: 3 2 3 15/30 Read block 516 len 28672 16/30 Write block 334 len 12288 data: 4 4 4 17/30 Write block 22 len 20480 data: 4 4 4 4 3 18/30 Read block 292 len 28672 19/30 Write block 308 len 40960 data: 6 5 6 6 5 5 3 3 4 3 20/30 Write block 447 len 40960 data: 6 3 5 5 5 6 6 6 6 6 21/30 Read block 80 len 4096 22/30 Write block 553 len 24576 data: 4 6 6 5 4 3 23/30 Flush 24/30 Write block 49 len 28672 data: 4 4 5 5 2 4 5 25/30 Write block 111 len 4096 data: 3 26/30 Write block 370 len 36864 data: 4 3 4 3 3 4 3 3 3 27/30 Read block 100 len 16384 28/30 Read block 316 len 8192 29/30 Write block 294 len 24576 data: 7 6 6 6 6 7 30/30 Write block 118 len 16384 data: 5 8 7 6 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.440944038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.441870302Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.441921656Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.441952863Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44197944Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442002618Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442024516Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442046393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442068281Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442088849Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442104867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442126235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442148093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442161861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.4421748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442187269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442199767Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442211886Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442230194Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442242863Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442255012Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44226705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442279029Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442290948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442302847Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442314676Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442326434Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442338333Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442350202Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44236218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442374729Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442386798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442398717Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442410606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442422534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442437363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442449441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44246126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442473039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442484768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442496726Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442520744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442533633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442545601Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44255752Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442569349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442581528Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442593716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442605705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442617824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442629792Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442641801Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44265393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442667959Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442680037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442692036Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442703905Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442715634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442727582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442740221Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44275227Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442764078Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442776897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442788796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442800745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442812524Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442824362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442836071Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44284796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442859839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442874037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442885996Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442897875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442909663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442921462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442933281Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44294516Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442957138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442969117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442981246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.442993225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443005273Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443017472Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443029621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44304157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443053548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443065507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443079286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443091164Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443103073Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443114942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443126821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44313863Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443150278Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443162267Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443174436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443186254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443198013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443209952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443221751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44323343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443245338Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443257067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443268906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443282395Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443294383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443306152Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443317841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443329659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443341418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443353357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443365146Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443377005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443388933Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443400792Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443412541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.44342425Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443436228Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443448247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443460296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443472265Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443485813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443498062Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:26.443510131Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1392,"session_id":"f166ac8b-75fd-4ab2-826e-35910fcf109e"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 49 49 49 82 82 82 T T T 3 060-079 50 50 50 85 85 85 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 44 44 44 75 75 75 T T T 6 120-139 49 49 49 82 82 82 T T T 7 140-159 50 50 50 86 86 86 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 49 49 49 82 82 82 F F F 10 200-219 49 49 49 82 82 82 F F F 11 220-239 47 47 47 79 79 79 F F F 12 240-259 47 47 47 79 79 79 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 50 50 50 85 85 85 T T T 15 300-319 50 50 50 86 86 86 F F F 16 320-339 50 50 50 86 86 86 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 46 46 46 77 77 77 T T T 19 380-399 40 40 40 66 66 66 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 50 50 50 83 83 83 F F F 22 440-459 50 50 50 86 86 86 F F F 23 460-479 50 50 50 85 85 85 F F F 24 480-499 44 44 44 75 75 75 F F F 25 500-519 46 46 46 76 76 76 F F F 26 520-539 50 50 50 85 85 85 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 84 84 84 F F F 29 580-599 44 44 44 75 75 75 F F F Max gen: 50, Max flush: 86 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 51 51 51 87 87 87 F F F 3 060-079 50 50 50 85 85 85 F F F 4 080-099 36 36 36 61 61 61 F F F 5 100-119 51 51 51 87 87 87 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 49 49 49 82 82 82 F F F 10 200-219 49 49 49 82 82 82 F F F 11 220-239 47 47 47 79 79 79 F F F 12 240-259 47 47 47 79 79 79 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 51 51 51 87 87 87 F F F 15 300-319 50 50 50 86 86 86 F F F 16 320-339 50 50 50 86 86 86 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 51 51 51 87 87 87 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 50 50 50 83 83 83 F F F 22 440-459 50 50 50 86 86 86 F F F 23 460-479 50 50 50 85 85 85 F F F 24 480-499 44 44 44 75 75 75 F F F 25 500-519 46 46 46 76 76 76 F F F 26 520-539 50 50 50 85 85 85 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 84 84 84 F F F 29 580-599 44 44 44 75 75 75 F F F Max gen: 51, Max flush: 87 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.712650157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.757055696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.757639525Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.757674141Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.75768813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.757702669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} {"msg":"Crucible ac264db5-fcda-464c-8a66-bf90fe6d69fc has session id: ca97d158-26cc-4d86-940e-c43ad87e859c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.758306286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Upstairs opts: Upstairs UUID: ac264db5-fcda-464c-8a66-bf90fe6d69fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.758340512Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Crucible stats registered with UUID: ac264db5-fcda-464c-8a66-bf90fe6d69fc","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.75835551Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:30.758368349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.758873907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.758898924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} {"msg":"ac264db5-fcda-464c-8a66-bf90fe6d69fc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.758961477Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.758993134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"0","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759024101Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"0","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759039929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"1","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759064077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"1","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759090884Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"2","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759118051Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"2","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759215041Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759250337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759274685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57463)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759299242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46432)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.7593249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50502)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759353646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759930316Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"2","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.759972442Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"0","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.760580409Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"2","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.760734282Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"0","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.761217332Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.761245339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.761405043Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"1","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.761932988Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"1","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[0]R flush_numbers[0..12]: [83, 86, 87, 85, 61, 87, 87, 86, 75, 82, 82, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762471691Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[0]R generation[0..12]: [50, 50, 51, 50, 36, 51, 51, 50, 44, 49, 49, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762497129Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762513227Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[1]R flush_numbers[0..12]: [83, 86, 87, 85, 61, 87, 87, 86, 75, 82, 82, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762527395Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[1]R generation[0..12]: [50, 50, 51, 50, 36, 51, 51, 50, 44, 49, 49, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762541974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762556103Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[2]R flush_numbers[0..12]: [83, 86, 87, 85, 61, 87, 87, 86, 75, 82, 82, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762569641Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[2]R generation[0..12]: [50, 50, 51, 50, 36, 51, 51, 50, 44, 49, 49, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762583909Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762597808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762620506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762635754Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762648803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762661492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"":"downstairs","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.76267474Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762686619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"0","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762699717Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"1","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762712606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"client":"2","":"downstairs","session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762725435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762737464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"ac264db5-fcda-464c-8a66-bf90fe6d69fc is now active with session: ca97d158-26cc-4d86-940e-c43ad87e859c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762749962Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.762762451Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400,"session_id":"ca97d158-26cc-4d86-940e-c43ad87e859c"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:30.7627753Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} {"msg":"Activated sub_volume ac264db5-fcda-464c-8a66-bf90fe6d69fc","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:01:30.762786348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 239 len 20480 data: 6 4 8 8 8 02/30 Write block 222 len 12288 data: 6 4 5 03/30 Flush 04/30 Flush 05/30 Write block 84 len 36864 data: 6 5 4 5 4 3 2 2 3 06/30 Write block 302 len 36864 data: 6 6 7 7 5 6 7 6 7 07/30 Read block 474 len 24576 08/30 Read block 395 len 4096 09/30 Read block 294 len 16384 10/30 Write block 402 len 24576 data: 4 4 3 3 2 2 11/30 Write block 315 len 12288 data: 4 5 4 12/30 Write block 263 len 12288 data: 3 3 4 13/30 Read block 484 len 20480 14/30 Write block 264 len 12288 data: 4 5 4 15/30 Read block 167 len 12288 16/30 Write block 101 len 32768 data: 3 5 3 3 3 3 3 3 17/30 Write block 62 len 32768 data: 6 6 5 4 3 3 4 4 18/30 Flush 19/30 Write block 62 len 4096 data: 7 20/30 Read block 54 len 16384 21/30 Write block 496 len 32768 data: 5 5 4 4 4 5 7 6 22/30 Write block 267 len 32768 data: 4 5 5 5 5 4 4 5 23/30 Read block 457 len 4096 24/30 Read block 79 len 36864 25/30 Write block 579 len 12288 data: 5 4 5 26/30 Read block 160 len 4096 27/30 Write block 361 len 28672 data: 7 6 5 4 4 3 3 28/30 Write block 441 len 12288 data: 5 4 5 29/30 Read block 432 len 36864 30/30 Write block 22 len 8192 data: 5 5 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 9 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:30.782276975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 50 50 50 86 86 86 T T T 2 040-059 51 51 51 87 87 87 F F F 3 060-079 52 52 52 90 90 90 T T T 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 49 49 49 82 82 82 F F F 10 200-219 49 49 49 82 82 82 F F F 11 220-239 52 52 52 88 88 88 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 52 52 52 90 90 90 T T T 14 280-299 51 51 51 87 87 87 F F F 15 300-319 52 52 52 90 90 90 F F F 16 320-339 50 50 50 86 86 86 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 51 51 51 87 87 87 T T T 19 380-399 40 40 40 66 66 66 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 50 50 50 83 83 83 F F F 22 440-459 50 50 50 86 86 86 T T T 23 460-479 50 50 50 85 85 85 F F F 24 480-499 44 44 44 75 75 75 T T T 25 500-519 46 46 46 76 76 76 T T T 26 520-539 50 50 50 85 85 85 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 84 84 84 T T T 29 580-599 44 44 44 75 75 75 T T T Max gen: 52, Max flush: 90 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 53 53 53 91 91 91 F F F 2 040-059 51 51 51 87 87 87 F F F 3 060-079 53 53 53 91 91 91 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 49 49 49 82 82 82 F F F 10 200-219 49 49 49 82 82 82 F F F 11 220-239 52 52 52 88 88 88 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 51 51 51 87 87 87 F F F 15 300-319 52 52 52 90 90 90 F F F 16 320-339 50 50 50 86 86 86 F F F 17 340-359 46 46 46 77 77 77 F F F 18 360-379 53 53 53 91 91 91 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 50 50 50 83 83 83 F F F 22 440-459 53 53 53 91 91 91 F F F 23 460-479 50 50 50 85 85 85 F F F 24 480-499 53 53 53 91 91 91 F F F 25 500-519 53 53 53 91 91 91 F F F 26 520-539 50 50 50 85 85 85 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 53 53 53 91 91 91 F F F 29 580-599 53 53 53 91 91 91 F F F Max gen: 53, Max flush: 91 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.041806934Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.085898196Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.086633489Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.086672065Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.086687623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.086702322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} {"msg":"Crucible f3017989-af65-423a-8250-af804ba6b621 has session id: be28caec-d369-4555-8a20-495c2a8d357f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087182922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Upstairs opts: Upstairs UUID: f3017989-af65-423a-8250-af804ba6b621, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087242886Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Crucible stats registered with UUID: f3017989-af65-423a-8250-af804ba6b621","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087269983Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:35.08729238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087699348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087725175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} {"msg":"f3017989-af65-423a-8250-af804ba6b621 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087791418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087809926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"0","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087841643Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"0","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.08786675Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"1","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087893688Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"1","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087919585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"2","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087946332Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"2","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087970739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.087998487Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.088025224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33633)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.088053521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50828)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.088075089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46234)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.088096996Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.08863315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"2","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.088794974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"0","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.089137628Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"1","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.089181613Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"2","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.089418379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"0","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.089734626Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.089864682Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"1","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.089992639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[0]R flush_numbers[0..12]: [83, 91, 87, 91, 90, 90, 87, 86, 75, 82, 82, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090389447Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[0]R generation[0..12]: [50, 53, 51, 53, 52, 52, 51, 50, 44, 49, 49, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090411285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090426614Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[1]R flush_numbers[0..12]: [83, 91, 87, 91, 90, 90, 87, 86, 75, 82, 82, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090444701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[1]R generation[0..12]: [50, 53, 51, 53, 52, 52, 51, 50, 44, 49, 49, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.09045915Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090473709Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[2]R flush_numbers[0..12]: [83, 91, 87, 91, 90, 90, 87, 86, 75, 82, 82, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090487257Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"[2]R generation[0..12]: [50, 53, 51, 53, 52, 52, 51, 50, 44, 49, 49, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090500656Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090520174Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090540791Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.0905604Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090576608Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090597736Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"":"downstairs","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090622563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.09064646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"0","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090670788Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"1","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090685496Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"client":"2","":"downstairs","session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090701295Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:01:35.090727342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"f3017989-af65-423a-8250-af804ba6b621 is now active with session: be28caec-d369-4555-8a20-495c2a8d357f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.09075086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090770907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408,"session_id":"be28caec-d369-4555-8a20-495c2a8d357f"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090791296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} {"msg":"Activated sub_volume f3017989-af65-423a-8250-af804ba6b621","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:35.090811693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 40960 data: 3 4 3 4 4 4 3 4 4 4 02/30 Write block 426 len 8192 data: 6 6 03/30 Read block 311 len 16384 04/30 Read block 294 len 16384 05/30 Write block 430 len 36864 data: 6 8 9 8 9 6 5 4 6 06/30 Read block 389 len 32768 07/30 Write block 289 len 12288 data: 9 11 11 08/30 Read block 65 len 40960 09/30 Write block 75 len 20480 data: 6 6 5 5 6 10/30 Read block 11 len 36864 11/30 Flush 12/30 Write block 524 len 12288 data: 6 5 5 13/30 Write block 549 len 4096 data: 4 14/30 Read block 196 len 4096 15/30 Read block 334 len 20480 16/30 Read block 580 len 40960 17/30 Write block 562 len 24576 data: 3 4 4 5 5 6 18/30 Write block 352 len 28672 data: 6 7 7 8 7 8 7 19/30 Read block 227 len 20480 20/30 Read block 47 len 32768 21/30 Write block 148 len 8192 data: 5 5 22/30 Write block 437 len 40960 data: 5 7 7 7 6 5 6 5 7 6 23/30 Write block 215 len 4096 data: 4 24/30 Read block 400 len 4096 25/30 Read block 411 len 16384 26/30 Read block 59 len 8192 27/30 Read block 164 len 8192 28/30 Read block 390 len 20480 29/30 Read block 317 len 12288 30/30 Read block 333 len 32768 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:35.111058132Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 53 53 53 91 91 91 F F F 2 040-059 51 51 51 87 87 87 F F F 3 060-079 54 54 54 92 92 92 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 54 54 54 92 92 92 T T T 8 160-179 44 44 44 75 75 75 F F F 9 180-199 49 49 49 82 82 82 F F F 10 200-219 49 49 49 82 82 82 T T T 11 220-239 52 52 52 88 88 88 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 54 54 54 92 92 92 F F F 15 300-319 52 52 52 90 90 90 F F F 16 320-339 50 50 50 86 86 86 F F F 17 340-359 46 46 46 77 77 77 T T T 18 360-379 53 53 53 91 91 91 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 54 54 54 92 92 92 T T T 22 440-459 53 53 53 91 91 91 T T T 23 460-479 50 50 50 85 85 85 F F F 24 480-499 53 53 53 91 91 91 F F F 25 500-519 53 53 53 91 91 91 F F F 26 520-539 50 50 50 85 85 85 T T T 27 540-559 50 50 50 86 86 86 T T T 28 560-579 53 53 53 91 91 91 T T T 29 580-599 53 53 53 91 91 91 F F F Max gen: 54, Max flush: 92 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 83 83 83 F F F 1 020-039 53 53 53 91 91 91 F F F 2 040-059 51 51 51 87 87 87 F F F 3 060-079 54 54 54 92 92 92 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 55 55 55 93 93 93 F F F 8 160-179 44 44 44 75 75 75 F F F 9 180-199 49 49 49 82 82 82 F F F 10 200-219 55 55 55 93 93 93 F F F 11 220-239 52 52 52 88 88 88 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 54 54 54 92 92 92 F F F 15 300-319 52 52 52 90 90 90 F F F 16 320-339 50 50 50 86 86 86 F F F 17 340-359 55 55 55 93 93 93 F F F 18 360-379 53 53 53 91 91 91 F F F 19 380-399 40 40 40 66 66 66 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 55 55 55 93 93 93 F F F 22 440-459 55 55 55 93 93 93 F F F 23 460-479 50 50 50 85 85 85 F F F 24 480-499 53 53 53 91 91 91 F F F 25 500-519 53 53 53 91 91 91 F F F 26 520-539 55 55 55 93 93 93 F F F 27 540-559 55 55 55 93 93 93 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 53 53 53 91 91 91 F F F Max gen: 55, Max flush: 93 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.362711337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.406446116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.407091829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.407125865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.407139904Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.407154492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} {"msg":"Crucible a285e3d2-48e3-4ede-9d65-ead55ed702c9 has session id: 403433ef-72de-4f49-b5c2-fc4215b2af38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.407639311Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Upstairs opts: Upstairs UUID: a285e3d2-48e3-4ede-9d65-ead55ed702c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.407681817Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Crucible stats registered with UUID: a285e3d2-48e3-4ede-9d65-ead55ed702c9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.407705145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:49.407741201Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408166606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408193094Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} {"msg":"a285e3d2-48e3-4ede-9d65-ead55ed702c9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408263246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408281754Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"0","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408310191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"0","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408328409Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"1","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408343848Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"1","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408357157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"2","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408375805Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"2","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408447247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408471095Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.40851508Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"ds_connection connected from Ok(127.0.0.1:55952)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408548866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"ds_connection connected from Ok(127.0.0.1:50066)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408576004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"ds_connection connected from Ok(127.0.0.1:53746)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.408600661Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.409242314Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"1","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.409271541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"0","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.409905275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"0","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.410058599Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"1","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.410447009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.410545198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.410572175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"2","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411109779Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"2","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[0]R flush_numbers[0..12]: [83, 91, 87, 92, 90, 90, 87, 93, 75, 82, 93, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411618946Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[0]R generation[0..12]: [50, 53, 51, 54, 52, 52, 51, 55, 44, 49, 55, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411640004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411654873Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[1]R flush_numbers[0..12]: [83, 91, 87, 92, 90, 90, 87, 93, 75, 82, 93, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411668661Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[1]R generation[0..12]: [50, 53, 51, 54, 52, 52, 51, 55, 44, 49, 55, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.41168233Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411696168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[2]R flush_numbers[0..12]: [83, 91, 87, 92, 90, 90, 87, 93, 75, 82, 93, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411709807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[2]R generation[0..12]: [50, 53, 51, 54, 52, 52, 51, 55, 44, 49, 55, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411723215Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411750383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411763261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411780789Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411794068Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411806977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"":"downstairs","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411819836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:01:49.411831514Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"0","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411852362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"1","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411865391Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"client":"2","":"downstairs","session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411878319Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411890208Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"a285e3d2-48e3-4ede-9d65-ead55ed702c9 is now active with session: 403433ef-72de-4f49-b5c2-fc4215b2af38","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411902727Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411914586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416,"session_id":"403433ef-72de-4f49-b5c2-fc4215b2af38"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411926604Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} {"msg":"Activated sub_volume a285e3d2-48e3-4ede-9d65-ead55ed702c9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:49.411937543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 190 len 24576 data: 7 8 7 7 7 6 02/30 Read block 274 len 32768 03/30 Read block 55 len 40960 04/30 Read block 382 len 24576 05/30 Write block 540 len 28672 data: 4 5 5 3 4 5 5 06/30 Write block 447 len 28672 data: 7 4 6 6 6 7 7 07/30 Read block 206 len 28672 08/30 Read block 275 len 20480 09/30 Read block 240 len 36864 10/30 Read block 293 len 28672 11/30 Read block 430 len 24576 12/30 Write block 287 len 8192 data: 9 8 13/30 Write block 377 len 28672 data: 4 4 4 7 8 8 8 14/30 Read block 443 len 40960 15/30 Write block 199 len 36864 data: 5 5 5 4 3 5 5 5 5 16/30 Read block 590 len 24576 17/30 Read block 400 len 16384 18/30 Read block 115 len 16384 19/30 Write block 392 len 4096 data: 8 20/30 Write block 546 len 24576 data: 6 5 5 5 3 3 21/30 Write block 0 len 32768 data: 3 4 4 4 5 5 7 7 22/30 Flush 23/30 Write block 310 len 24576 data: 8 7 6 6 4 5 24/30 Write block 487 len 24576 data: 6 6 6 6 3 3 25/30 Write block 465 len 24576 data: 3 2 2 2 2 2 26/30 Write block 177 len 8192 data: 9 6 27/30 Read block 492 len 4096 28/30 Write block 177 len 40960 data: 10 7 6 3 3 4 3 5 4 4 29/30 Read block 238 len 8192 30/30 Read block 210 len 36864 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:49.433648839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 94 94 94 F F F 1 020-039 53 53 53 91 91 91 F F F 2 040-059 51 51 51 87 87 87 F F F 3 060-079 54 54 54 92 92 92 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 55 55 55 93 93 93 F F F 8 160-179 44 44 44 75 75 75 T T T 9 180-199 56 56 56 94 94 94 T T T 10 200-219 56 56 56 94 94 94 F F F 11 220-239 52 52 52 88 88 88 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 56 56 56 94 94 94 F F F 15 300-319 52 52 52 90 90 90 T T T 16 320-339 50 50 50 86 86 86 F F F 17 340-359 55 55 55 93 93 93 F F F 18 360-379 56 56 56 94 94 94 F F F 19 380-399 56 56 56 94 94 94 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 55 55 55 93 93 93 F F F 22 440-459 56 56 56 94 94 94 F F F 23 460-479 50 50 50 85 85 85 T T T 24 480-499 53 53 53 91 91 91 T T T 25 500-519 53 53 53 91 91 91 F F F 26 520-539 55 55 55 93 93 93 F F F 27 540-559 56 56 56 94 94 94 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 53 53 53 91 91 91 F F F Max gen: 56, Max flush: 94 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 94 94 94 F F F 1 020-039 53 53 53 91 91 91 F F F 2 040-059 51 51 51 87 87 87 F F F 3 060-079 54 54 54 92 92 92 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 55 55 55 93 93 93 F F F 8 160-179 57 57 57 95 95 95 F F F 9 180-199 57 57 57 95 95 95 F F F 10 200-219 56 56 56 94 94 94 F F F 11 220-239 52 52 52 88 88 88 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 53 53 53 91 91 91 F F F 14 280-299 56 56 56 94 94 94 F F F 15 300-319 57 57 57 95 95 95 F F F 16 320-339 50 50 50 86 86 86 F F F 17 340-359 55 55 55 93 93 93 F F F 18 360-379 56 56 56 94 94 94 F F F 19 380-399 56 56 56 94 94 94 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 55 55 55 93 93 93 F F F 22 440-459 56 56 56 94 94 94 F F F 23 460-479 57 57 57 95 95 95 F F F 24 480-499 57 57 57 95 95 95 F F F 25 500-519 53 53 53 91 91 91 F F F 26 520-539 55 55 55 93 93 93 F F F 27 540-559 56 56 56 94 94 94 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 53 53 53 91 91 91 F F F Max gen: 57, Max flush: 95 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.670292903Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.715175143Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.715820655Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.71587125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.715893208Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.715917355Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} {"msg":"Crucible dfcf4b5f-3171-4324-a42f-3be31e68cccd has session id: 37f78101-d2f2-483e-ab50-bdf648f64996","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.716464038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Upstairs opts: Upstairs UUID: dfcf4b5f-3171-4324-a42f-3be31e68cccd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.716510113Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Crucible stats registered with UUID: dfcf4b5f-3171-4324-a42f-3be31e68cccd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.71654126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:53.716564087Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.716982464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717014651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} {"msg":"dfcf4b5f-3171-4324-a42f-3be31e68cccd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717069185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717097262Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"0","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717128219Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"0","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717142777Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"1","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717157156Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"1","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717170214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"2","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717183863Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"2","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717197481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.71721099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717234048Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"ds_connection connected from Ok(127.0.0.1:33232)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717302501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"ds_connection connected from Ok(127.0.0.1:36659)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717349246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"ds_connection connected from Ok(127.0.0.1:51274)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717546895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.717995618Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"2","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.718036024Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"0","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.718130474Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"1","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.718628742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"2","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.718659089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"0","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.718797225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"1","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719208502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719295093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[0]R flush_numbers[0..12]: [94, 91, 87, 92, 90, 90, 87, 93, 95, 95, 94, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719405051Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[0]R generation[0..12]: [56, 53, 51, 54, 52, 52, 51, 55, 57, 57, 56, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719425499Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719440108Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[1]R flush_numbers[0..12]: [94, 91, 87, 92, 90, 90, 87, 93, 95, 95, 94, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719453726Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[1]R generation[0..12]: [56, 53, 51, 54, 52, 52, 51, 55, 57, 57, 56, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719467205Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719485713Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[2]R flush_numbers[0..12]: [94, 91, 87, 92, 90, 90, 87, 93, 95, 95, 94, 88]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719499621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[2]R generation[0..12]: [56, 53, 51, 54, 52, 52, 51, 55, 57, 57, 56, 52]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.71951339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719527578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719540587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719553756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719566385Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719584852Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"":"downstairs","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719597991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.71960986Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"0","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719622839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"1","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719635777Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"client":"2","":"downstairs","session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719648746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719660765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"dfcf4b5f-3171-4324-a42f-3be31e68cccd is now active with session: 37f78101-d2f2-483e-ab50-bdf648f64996","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719674383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719688262Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424,"session_id":"37f78101-d2f2-483e-ab50-bdf648f64996"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.71970056Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} {"msg":"Activated sub_volume dfcf4b5f-3171-4324-a42f-3be31e68cccd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:53.719711539Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 202 len 32768 data: 5 4 6 6 6 6 5 5 02/30 Write block 468 len 12288 data: 3 3 3 03/30 Read block 163 len 24576 04/30 Write block 231 len 32768 data: 7 5 4 3 4 5 6 5 05/30 Write block 65 len 4096 data: 5 06/30 Write block 364 len 36864 data: 5 5 4 4 2 3 5 4 5 07/30 Flush 08/30 Flush 09/30 Write block 270 len 8192 data: 6 6 10/30 Write block 37 len 16384 data: 6 6 5 5 11/30 Write block 492 len 24576 data: 4 4 4 4 6 6 12/30 Read block 495 len 40960 13/30 Write block 444 len 12288 data: 6 8 7 14/30 Write block 55 len 4096 data: 6 15/30 Write block 321 len 36864 data: 3 4 4 5 5 5 6 6 7 16/30 Flush 17/30 Write block 158 len 16384 data: 7 7 7 5 18/30 Read block 371 len 8192 19/30 Flush 20/30 Write block 54 len 8192 data: 5 7 21/30 Read block 461 len 32768 22/30 Read block 390 len 12288 23/30 Read block 79 len 32768 24/30 Write block 279 len 8192 data: 5 6 25/30 Read block 509 len 32768 26/30 Read block 449 len 12288 27/30 Write block 39 len 36864 data: 6 6 6 6 7 7 4 4 4 28/30 Read block 458 len 32768 29/30 Write block 585 len 28672 data: 6 5 6 5 5 4 3 30/30 Read block 108 len 36864 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 10 9 11 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:53.739930341Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 94 94 94 F F F 1 020-039 58 58 58 98 98 98 T T T 2 040-059 58 58 58 98 98 98 T T T 3 060-079 58 58 58 96 96 96 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 58 58 58 99 99 99 F F F 8 160-179 58 58 58 99 99 99 F F F 9 180-199 57 57 57 95 95 95 F F F 10 200-219 58 58 58 96 96 96 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 58 58 58 98 98 98 T T T 14 280-299 56 56 56 94 94 94 T T T 15 300-319 57 57 57 95 95 95 F F F 16 320-339 58 58 58 98 98 98 F F F 17 340-359 55 55 55 93 93 93 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 56 56 56 94 94 94 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 55 55 55 93 93 93 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 58 58 58 96 96 96 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 53 53 53 91 91 91 F F F 26 520-539 55 55 55 93 93 93 F F F 27 540-559 56 56 56 94 94 94 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 53 53 53 91 91 91 T T T Max gen: 58, Max flush: 99 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 94 94 94 F F F 1 020-039 59 59 59 100 100 100 F F F 2 040-059 59 59 59 100 100 100 F F F 3 060-079 58 58 58 96 96 96 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 58 58 58 99 99 99 F F F 8 160-179 58 58 58 99 99 99 F F F 9 180-199 57 57 57 95 95 95 F F F 10 200-219 58 58 58 96 96 96 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 59 59 59 100 100 100 F F F 15 300-319 57 57 57 95 95 95 F F F 16 320-339 58 58 58 98 98 98 F F F 17 340-359 55 55 55 93 93 93 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 56 56 56 94 94 94 F F F 20 400-419 52 52 52 90 90 90 F F F 21 420-439 55 55 55 93 93 93 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 58 58 58 96 96 96 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 53 53 53 91 91 91 F F F 26 520-539 55 55 55 93 93 93 F F F 27 540-559 56 56 56 94 94 94 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 59 59 59 100 100 100 F F F Max gen: 59, Max flush: 100 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:57.980306694Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.024580647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.025299412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.025336259Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.025355766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.025370635Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} {"msg":"Crucible c114b92a-7b0b-43f5-b7d6-469027734d6d has session id: fc99caf4-7e3e-440f-bee3-8437ace56279","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.025935156Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Upstairs opts: Upstairs UUID: c114b92a-7b0b-43f5-b7d6-469027734d6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.025977492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Crucible stats registered with UUID: c114b92a-7b0b-43f5-b7d6-469027734d6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.02599465Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:58.026007989Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026442433Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.0264758Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} {"msg":"c114b92a-7b0b-43f5-b7d6-469027734d6d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.02656531Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026596947Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"0","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026644532Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"0","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026683178Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"1","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026714195Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"1","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026740292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"2","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026774058Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"2","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.026990006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.027054199Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.027078837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"ds_connection connected from Ok(127.0.0.1:51647)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.027101504Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"ds_connection connected from Ok(127.0.0.1:38597)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.027125122Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"ds_connection connected from Ok(127.0.0.1:57957)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.027149039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.027570506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"2","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.027613441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"0","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.028254554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"0","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.028289331Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"2","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.028423697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"1","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.028942223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.028978629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029013875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"1","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[0]R flush_numbers[0..12]: [94, 100, 100, 96, 90, 90, 87, 99, 99, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029562598Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[0]R generation[0..12]: [56, 59, 59, 58, 52, 52, 51, 58, 58, 57, 58, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029598554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029625231Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[1]R flush_numbers[0..12]: [94, 100, 100, 96, 90, 90, 87, 99, 99, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029651498Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[1]R generation[0..12]: [56, 59, 59, 58, 52, 52, 51, 58, 58, 57, 58, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029675876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029701593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[2]R flush_numbers[0..12]: [94, 100, 100, 96, 90, 90, 87, 99, 99, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.02972929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[2]R generation[0..12]: [56, 59, 59, 58, 52, 52, 51, 58, 58, 57, 58, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029758367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029787294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029822001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029850618Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029877615Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029904532Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"":"downstairs","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029936089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029962336Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"0","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.029989323Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"1","":"downstairs","session_id":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.03002387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"client":"2","":"downstairs","session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.030045897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.030059056Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"c114b92a-7b0b-43f5-b7d6-469027734d6d is now active with session: fc99caf4-7e3e-440f-bee3-8437ace56279","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.030072065Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.030084093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432,"session_id":"fc99caf4-7e3e-440f-bee3-8437ace56279"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.030096082Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} {"msg":"Activated sub_volume c114b92a-7b0b-43f5-b7d6-469027734d6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:01:58.030107281Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 10 len 36864 data: 8 9 9 7 8 8 8 7 7 03/30 Write block 61 len 4096 data: 7 04/30 Read block 14 len 24576 05/30 Write block 299 len 24576 data: 8 6 5 7 7 8 06/30 Read block 37 len 16384 07/30 Flush 08/30 Write block 398 len 36864 data: 6 7 5 4 5 5 4 4 3 09/30 Read block 435 len 36864 10/30 Write block 550 len 24576 data: 4 4 4 5 7 7 11/30 Write block 81 len 24576 data: 6 6 7 7 6 5 12/30 Read block 329 len 16384 13/30 Read block 319 len 24576 14/30 Read block 532 len 8192 15/30 Read block 276 len 16384 16/30 Write block 35 len 16384 data: 5 6 7 7 17/30 Read block 253 len 40960 18/30 Write block 353 len 4096 data: 8 19/30 Read block 47 len 8192 20/30 Write block 81 len 20480 data: 7 7 8 8 7 21/30 Flush 22/30 Write block 109 len 36864 data: 3 3 4 3 2 2 4 4 5 23/30 Read block 342 len 32768 24/30 Read block 8 len 28672 25/30 Flush 26/30 Write block 79 len 28672 data: 7 5 8 8 9 9 8 27/30 Write block 118 len 24576 data: 6 9 8 7 7 5 28/30 Write block 518 len 8192 data: 4 4 29/30 Read block 272 len 8192 30/30 Write block 525 len 16384 data: 6 6 5 6 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:01:58.050148181Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 F F F 1 020-039 60 60 60 103 103 103 F F F 2 040-059 59 59 59 100 100 100 F F F 3 060-079 60 60 60 102 102 102 T T T 4 080-099 60 60 60 103 103 103 T T T 5 100-119 60 60 60 104 104 104 T T T 6 120-139 51 51 51 87 87 87 T T T 7 140-159 58 58 58 99 99 99 F F F 8 160-179 58 58 58 99 99 99 F F F 9 180-199 57 57 57 95 95 95 F F F 10 200-219 58 58 58 96 96 96 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 102 102 102 F F F 16 320-339 58 58 58 98 98 98 F F F 17 340-359 60 60 60 103 103 103 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 60 60 60 103 103 103 F F F 21 420-439 55 55 55 93 93 93 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 58 58 58 96 96 96 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 53 53 53 91 91 91 T T T 26 520-539 55 55 55 93 93 93 T T T 27 540-559 60 60 60 103 103 103 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 59 59 59 100 100 100 F F F Max gen: 60, Max flush: 104 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 F F F 1 020-039 60 60 60 103 103 103 F F F 2 040-059 59 59 59 100 100 100 F F F 3 060-079 61 61 61 105 105 105 F F F 4 080-099 61 61 61 105 105 105 F F F 5 100-119 61 61 61 105 105 105 F F F 6 120-139 61 61 61 105 105 105 F F F 7 140-159 58 58 58 99 99 99 F F F 8 160-179 58 58 58 99 99 99 F F F 9 180-199 57 57 57 95 95 95 F F F 10 200-219 58 58 58 96 96 96 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 102 102 102 F F F 16 320-339 58 58 58 98 98 98 F F F 17 340-359 60 60 60 103 103 103 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 60 60 60 103 103 103 F F F 20 400-419 60 60 60 103 103 103 F F F 21 420-439 55 55 55 93 93 93 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 58 58 58 96 96 96 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 61 61 61 105 105 105 F F F 26 520-539 61 61 61 105 105 105 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 59 59 59 100 100 100 F F F Max gen: 61, Max flush: 105 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.28870975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.332779904Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.333429356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.333480321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.333504919Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.333529526Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} {"msg":"Crucible df1d0778-cca3-4168-ae1b-830ce42cc7ed has session id: 3cd7a125-a31a-49fc-8ba5-332ffdcab90b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334030304Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Upstairs opts: Upstairs UUID: df1d0778-cca3-4168-ae1b-830ce42cc7ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334076149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Crucible stats registered with UUID: df1d0778-cca3-4168-ae1b-830ce42cc7ed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334100626Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:12.334121964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334656918Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334680976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} {"msg":"df1d0778-cca3-4168-ae1b-830ce42cc7ed active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334761827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334783815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"0","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334811862Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"0","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334825841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334840279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334853438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"2","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334867936Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"2","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334936149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.334973885Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.335063226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38659)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.335096002Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43007)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.33512509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:12.335191403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:12.335228279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:12.335254776Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.335419319Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.335856733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"2","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.33588467Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"0","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.336531533Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"0","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.336574238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"2","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.337194744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:12.337271606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.336122021Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37829)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.336269466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.337294299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.337851421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[0]R flush_numbers[0..12]: [102, 103, 100, 105, 105, 105, 105, 99, 99, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338487815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[0]R generation[0..12]: [60, 60, 59, 61, 61, 61, 61, 58, 58, 57, 58, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338513202Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.33852916Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[1]R flush_numbers[0..12]: [102, 103, 100, 105, 105, 105, 105, 99, 99, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338549768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[1]R generation[0..12]: [60, 60, 59, 61, 61, 61, 61, 58, 58, 57, 58, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338564096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338578435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[2]R flush_numbers[0..12]: [102, 103, 100, 105, 105, 105, 105, 99, 99, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338592613Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[2]R generation[0..12]: [60, 60, 59, 61, 61, 61, 61, 58, 58, 57, 58, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338606202Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338620091Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338632849Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338645858Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338658757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338671685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"":"downstairs","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338684704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338696463Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"0","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338709601Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"1","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.33872256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"client":"2","":"downstairs","session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338743378Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338758216Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"df1d0778-cca3-4168-ae1b-830ce42cc7ed is now active with session: 3cd7a125-a31a-49fc-8ba5-332ffdcab90b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338771225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338783084Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440,"session_id":"3cd7a125-a31a-49fc-8ba5-332ffdcab90b"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338795023Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} {"msg":"Activated sub_volume df1d0778-cca3-4168-ae1b-830ce42cc7ed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:22.338806111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 415 len 28672 data: 4 4 7 8 8 6 5 02/30 Read block 359 len 28672 03/30 Write block 309 len 28672 data: 7 9 8 7 7 5 6 04/30 Read block 557 len 16384 05/30 Read block 191 len 4096 06/30 Read block 459 len 28672 07/30 Read block 159 len 4096 08/30 Read block 545 len 24576 09/30 Read block 370 len 20480 10/30 Read block 47 len 16384 11/30 Read block 548 len 28672 12/30 Write block 174 len 20480 data: 8 8 12 11 8 13/30 Read block 177 len 40960 14/30 Read block 399 len 28672 15/30 Read block 572 len 4096 16/30 Read block 398 len 8192 17/30 Read block 105 len 40960 18/30 Write block 218 len 8192 data: 4 4 19/30 Read block 302 len 4096 20/30 Read block 479 len 4096 21/30 Write block 187 len 8192 data: 6 7 22/30 Write block 211 len 28672 data: 5 4 6 6 5 4 4 23/30 Read block 267 len 12288 24/30 Write block 337 len 32768 data: 4 4 3 3 4 6 5 4 25/30 Read block 417 len 40960 26/30 Read block 208 len 16384 27/30 Read block 200 len 24576 28/30 Read block 394 len 12288 29/30 Write block 403 len 16384 data: 6 5 5 4 30/30 Write block 386 len 12288 data: 6 3 4 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:22.360230707Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 F F F 1 020-039 60 60 60 103 103 103 F F F 2 040-059 59 59 59 100 100 100 F F F 3 060-079 61 61 61 105 105 105 F F F 4 080-099 61 61 61 105 105 105 F F F 5 100-119 61 61 61 105 105 105 F F F 6 120-139 61 61 61 105 105 105 F F F 7 140-159 58 58 58 99 99 99 F F F 8 160-179 58 58 58 99 99 99 T T T 9 180-199 57 57 57 95 95 95 T T T 10 200-219 58 58 58 96 96 96 T T T 11 220-239 58 58 58 96 96 96 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 60 60 60 102 102 102 T T T 16 320-339 58 58 58 98 98 98 T T T 17 340-359 60 60 60 103 103 103 T T T 18 360-379 58 58 58 96 96 96 F F F 19 380-399 60 60 60 103 103 103 T T T 20 400-419 60 60 60 103 103 103 T T T 21 420-439 55 55 55 93 93 93 T T T 22 440-459 58 58 58 98 98 98 F F F 23 460-479 58 58 58 96 96 96 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 61 61 61 105 105 105 F F F 26 520-539 61 61 61 105 105 105 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 59 59 59 100 100 100 F F F Max gen: 61, Max flush: 105 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 F F F 1 020-039 60 60 60 103 103 103 F F F 2 040-059 59 59 59 100 100 100 F F F 3 060-079 61 61 61 105 105 105 F F F 4 080-099 61 61 61 105 105 105 F F F 5 100-119 61 61 61 105 105 105 F F F 6 120-139 61 61 61 105 105 105 F F F 7 140-159 58 58 58 99 99 99 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 62 62 62 106 106 106 F F F 11 220-239 58 58 58 96 96 96 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 60 60 60 102 102 102 F F F 15 300-319 62 62 62 106 106 106 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 58 58 58 96 96 96 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 62 62 62 106 106 106 F F F 22 440-459 58 58 58 98 98 98 F F F 23 460-479 58 58 58 96 96 96 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 61 61 61 105 105 105 F F F 26 520-539 61 61 61 105 105 105 F F F 27 540-559 60 60 60 103 103 103 F F F 28 560-579 55 55 55 93 93 93 F F F 29 580-599 59 59 59 100 100 100 F F F Max gen: 62, Max flush: 106 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.622809326Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.668743646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.669474539Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.669520764Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.669538633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.669557581Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} {"msg":"Crucible 219b04c3-8360-4811-b567-b0499b28806a has session id: db673a10-eaba-4cd3-ad4f-b801fbd39ff4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670249348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Upstairs opts: Upstairs UUID: 219b04c3-8360-4811-b567-b0499b28806a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670288675Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Crucible stats registered with UUID: 219b04c3-8360-4811-b567-b0499b28806a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670305033Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:26.670317981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670763795Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670787212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} {"msg":"219b04c3-8360-4811-b567-b0499b28806a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670854785Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670873263Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"0","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670901381Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"0","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670915519Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"1","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670929648Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"1","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670942816Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"2","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670956685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"2","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670969553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.670982862Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.67100118Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52032)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.671201599Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"ds_connection connected from Ok(127.0.0.1:59375)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.671242575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47012)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.671272602Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.671800357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"0","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.671840763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"1","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.671876669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"2","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.672428441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"0","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.672550529Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"1","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.672692134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"2","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673070534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673163625Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[0]R flush_numbers[0..12]: [102, 103, 100, 105, 105, 105, 105, 99, 106, 106, 106, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673279083Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[0]R generation[0..12]: [60, 60, 59, 61, 61, 61, 61, 58, 62, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673298661Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673313249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[1]R flush_numbers[0..12]: [102, 103, 100, 105, 105, 105, 105, 99, 106, 106, 106, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673327108Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[1]R generation[0..12]: [60, 60, 59, 61, 61, 61, 61, 58, 62, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673340556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673354465Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[2]R flush_numbers[0..12]: [102, 103, 100, 105, 105, 105, 105, 99, 106, 106, 106, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673367813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[2]R generation[0..12]: [60, 60, 59, 61, 61, 61, 61, 58, 62, 62, 62, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673385701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.67340003Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Max found gen is 63","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673412859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Generation requested: 64 >= found:63","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673425827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673438466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673451105Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673472552Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673484291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"0","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.67349746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"1","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673510448Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"client":"2","":"downstairs","session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673524247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673536166Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"219b04c3-8360-4811-b567-b0499b28806a is now active with session: db673a10-eaba-4cd3-ad4f-b801fbd39ff4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673548875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673560673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448,"session_id":"db673a10-eaba-4cd3-ad4f-b801fbd39ff4"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673572892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} {"msg":"Activated sub_volume 219b04c3-8360-4811-b567-b0499b28806a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:26.673583871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 436 len 16384 data: 6 6 8 8 02/30 Flush 03/30 Write block 472 len 24576 data: 4 4 4 5 5 6 04/30 Flush 05/30 Write block 401 len 24576 data: 5 6 7 6 6 5 06/30 Write block 394 len 4096 data: 6 07/30 Write block 19 len 40960 data: 6 6 3 6 6 5 5 4 3 3 08/30 Write block 391 len 24576 data: 6 9 7 7 4 4 09/30 Read block 570 len 8192 10/30 Read block 155 len 20480 11/30 Write block 448 len 12288 data: 5 7 7 12/30 Read block 325 len 40960 13/30 Read block 238 len 28672 14/30 Read block 205 len 40960 15/30 Write block 297 len 8192 data: 7 7 16/30 Write block 228 len 32768 data: 6 6 7 8 6 5 4 5 17/30 Read block 537 len 4096 18/30 Write block 476 len 24576 data: 6 7 6 6 6 6 19/30 Write block 378 len 40960 data: 5 5 8 9 9 9 8 6 7 4 20/30 Write block 16 len 8192 data: 9 8 21/30 Read block 580 len 8192 22/30 Read block 3 len 16384 23/30 Write block 536 len 16384 data: 5 5 5 4 24/30 Write block 124 len 40960 data: 6 7 6 7 7 5 4 3 3 3 25/30 Write block 514 len 16384 data: 5 4 3 4 26/30 Read block 343 len 20480 27/30 Write block 557 len 32768 data: 5 4 3 4 4 4 5 5 28/30 Write block 342 len 12288 data: 7 6 5 29/30 Read block 502 len 8192 30/30 Read block 484 len 40960 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 6 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 26 26 25 77 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:26.693553368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 102 102 102 T T T 1 020-039 60 60 60 103 103 103 T T T 2 040-059 59 59 59 100 100 100 F F F 3 060-079 61 61 61 105 105 105 F F F 4 080-099 61 61 61 105 105 105 F F F 5 100-119 61 61 61 105 105 105 F F F 6 120-139 61 61 61 105 105 105 T T T 7 140-159 58 58 58 99 99 99 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 62 62 62 106 106 106 F F F 11 220-239 58 58 58 96 96 96 T T T 12 240-259 52 52 52 88 88 88 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 60 60 60 102 102 102 T T T 15 300-319 62 62 62 106 106 106 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 62 62 62 106 106 106 T T T 18 360-379 58 58 58 96 96 96 T T T 19 380-399 62 62 62 106 106 106 T T T 20 400-419 62 62 62 106 106 106 T T T 21 420-439 64 64 64 107 107 107 F F F 22 440-459 58 58 58 98 98 98 T T T 23 460-479 64 64 64 108 108 108 T T T 24 480-499 58 58 58 98 98 98 T T T 25 500-519 61 61 61 105 105 105 T T T 26 520-539 61 61 61 105 105 105 T T T 27 540-559 60 60 60 103 103 103 T T T 28 560-579 55 55 55 93 93 93 T T T 29 580-599 59 59 59 100 100 100 F F F Max gen: 64, Max flush: 108 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 65 65 65 109 109 109 F F F 2 040-059 59 59 59 100 100 100 F F F 3 060-079 61 61 61 105 105 105 F F F 4 080-099 61 61 61 105 105 105 F F F 5 100-119 61 61 61 105 105 105 F F F 6 120-139 65 65 65 109 109 109 F F F 7 140-159 58 58 58 99 99 99 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 62 62 62 106 106 106 F F F 10 200-219 62 62 62 106 106 106 F F F 11 220-239 65 65 65 109 109 109 F F F 12 240-259 52 52 52 88 88 88 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 65 65 65 109 109 109 F F F 15 300-319 62 62 62 106 106 106 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 65 65 65 109 109 109 F F F 18 360-379 65 65 65 109 109 109 F F F 19 380-399 65 65 65 109 109 109 F F F 20 400-419 65 65 65 109 109 109 F F F 21 420-439 64 64 64 107 107 107 F F F 22 440-459 65 65 65 109 109 109 F F F 23 460-479 65 65 65 109 109 109 F F F 24 480-499 65 65 65 109 109 109 F F F 25 500-519 65 65 65 109 109 109 F F F 26 520-539 65 65 65 109 109 109 F F F 27 540-559 65 65 65 109 109 109 F F F 28 560-579 65 65 65 109 109 109 F F F 29 580-599 59 59 59 100 100 100 F F F Max gen: 65, Max flush: 109 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:40.990643372Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:41.032893337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.033265568Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.064171185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.064737356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.064772502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.064786681Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.064801089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} {"msg":"Crucible e35cd288-32bb-4a1b-b85a-b2c9a44b44b4 has session id: d761a1b1-7f6e-403d-9727-186727edc9d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065379439Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Upstairs opts: Upstairs UUID: e35cd288-32bb-4a1b-b85a-b2c9a44b44b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065423804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Crucible stats registered with UUID: e35cd288-32bb-4a1b-b85a-b2c9a44b44b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065445002Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:41.065458481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065823892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.06584813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} {"msg":"e35cd288-32bb-4a1b-b85a-b2c9a44b44b4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065915603Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065934281Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"0","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065962558Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"0","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065980266Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"1","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.065996834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"1","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066020352Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"2","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066043669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"2","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066115182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066155488Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066181855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38139)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066206232Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35016)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.0662321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42987)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066444338Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066863134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"1","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.066893861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"2","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.067099359Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"0","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.067528825Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"1","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.067564031Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"2","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.067593988Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"0","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068182746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068218653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[0]R flush_numbers[0..12]: [109, 109, 100, 105, 105, 105, 109, 99, 106, 106, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.06824693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[0]R generation[0..12]: [65, 65, 59, 61, 61, 61, 65, 58, 62, 62, 62, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068264708Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068279666Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[1]R flush_numbers[0..12]: [109, 109, 100, 105, 105, 105, 109, 99, 106, 106, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068293845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[1]R generation[0..12]: [65, 65, 59, 61, 61, 61, 65, 58, 62, 62, 62, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068307824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068321952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[2]R flush_numbers[0..12]: [109, 109, 100, 105, 105, 105, 109, 99, 106, 106, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068335501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[2]R generation[0..12]: [65, 65, 59, 61, 61, 61, 65, 58, 62, 62, 62, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068348809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068371337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068396634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068424001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068451558Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068465427Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"":"downstairs","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068479306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068491544Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"0","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068505093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"1","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068518462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"client":"2","":"downstairs","session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.06853159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068544079Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"e35cd288-32bb-4a1b-b85a-b2c9a44b44b4 is now active with session: d761a1b1-7f6e-403d-9727-186727edc9d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068556778Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068568466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456,"session_id":"d761a1b1-7f6e-403d-9727-186727edc9d0"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068580595Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} {"msg":"Activated sub_volume e35cd288-32bb-4a1b-b85a-b2c9a44b44b4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:41.068591674Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 106 len 40960 data: 4 4 4 4 4 5 4 3 3 5 03/30 Flush 04/30 Write block 195 len 20480 data: 7 7 5 4 6 05/30 Read block 338 len 40960 06/30 Write block 589 len 16384 data: 6 5 4 3 07/30 Write block 487 len 24576 data: 7 7 7 7 4 5 08/30 Write block 569 len 32768 data: 4 5 5 6 5 6 6 5 09/30 Read block 339 len 28672 10/30 Read block 298 len 12288 11/30 Write block 377 len 24576 data: 5 6 6 9 10 10 12/30 Read block 378 len 36864 13/30 Write block 59 len 20480 data: 9 8 8 8 7 14/30 Read block 517 len 40960 15/30 Write block 238 len 40960 data: 6 7 5 9 9 9 7 6 6 6 16/30 Write block 381 len 28672 data: 11 11 10 9 7 8 5 17/30 Write block 152 len 28672 data: 5 6 7 6 6 6 8 18/30 Read block 45 len 8192 19/30 Write block 456 len 36864 data: 7 7 5 5 5 5 4 3 3 20/30 Read block 154 len 32768 21/30 Flush 22/30 Write block 398 len 28672 data: 7 8 6 6 7 8 7 23/30 Read block 400 len 20480 24/30 Flush 25/30 Write block 39 len 12288 data: 7 7 7 26/30 Write block 213 len 24576 data: 7 7 6 5 5 5 27/30 Read block 186 len 12288 28/30 Read block 474 len 4096 29/30 Write block 437 len 28672 data: 7 9 9 8 7 6 7 30/30 Write block 427 len 16384 data: 7 7 7 7 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:41.090855302Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 65 65 65 109 109 109 T T T 2 040-059 66 66 66 112 112 112 T T T 3 060-079 66 66 66 112 112 112 F F F 4 080-099 61 61 61 105 105 105 F F F 5 100-119 66 66 66 111 111 111 F F F 6 120-139 65 65 65 109 109 109 F F F 7 140-159 66 66 66 112 112 112 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 66 66 66 112 112 112 F F F 10 200-219 62 62 62 106 106 106 T T T 11 220-239 66 66 66 112 112 112 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 65 65 65 109 109 109 F F F 15 300-319 62 62 62 106 106 106 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 65 65 65 109 109 109 F F F 18 360-379 66 66 66 112 112 112 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 66 66 66 113 113 113 F F F 21 420-439 64 64 64 107 107 107 T T T 22 440-459 66 66 66 112 112 112 T T T 23 460-479 66 66 66 112 112 112 F F F 24 480-499 66 66 66 112 112 112 F F F 25 500-519 65 65 65 109 109 109 F F F 26 520-539 65 65 65 109 109 109 F F F 27 540-559 65 65 65 109 109 109 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 66, Max flush: 113 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 67 67 67 114 114 114 F F F 2 040-059 67 67 67 114 114 114 F F F 3 060-079 66 66 66 112 112 112 F F F 4 080-099 61 61 61 105 105 105 F F F 5 100-119 66 66 66 111 111 111 F F F 6 120-139 65 65 65 109 109 109 F F F 7 140-159 66 66 66 112 112 112 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 66 66 66 112 112 112 F F F 10 200-219 67 67 67 114 114 114 F F F 11 220-239 66 66 66 112 112 112 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 59 59 59 100 100 100 F F F 14 280-299 65 65 65 109 109 109 F F F 15 300-319 62 62 62 106 106 106 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 65 65 65 109 109 109 F F F 18 360-379 66 66 66 112 112 112 F F F 19 380-399 66 66 66 113 113 113 F F F 20 400-419 66 66 66 113 113 113 F F F 21 420-439 67 67 67 114 114 114 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 66 66 66 112 112 112 F F F 25 500-519 65 65 65 109 109 109 F F F 26 520-539 65 65 65 109 109 109 F F F 27 540-559 65 65 65 109 109 109 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 67, Max flush: 114 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.360041672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.403603569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.404239363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.40427325Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.404287278Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.404301696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} {"msg":"Crucible e126ec39-3009-4d4b-ba3b-7f034568f589 has session id: d050313d-40a2-43b5-a59d-898bca1e3e55","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.404776687Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Upstairs opts: Upstairs UUID: e126ec39-3009-4d4b-ba3b-7f034568f589, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.404875167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Crucible stats registered with UUID: e126ec39-3009-4d4b-ba3b-7f034568f589","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.404912243Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:45.4049404Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405374875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405404252Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} {"msg":"e126ec39-3009-4d4b-ba3b-7f034568f589 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405462446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405489543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"0","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405522889Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"0","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405549006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"1","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405576703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"1","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405603401Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"2","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405629608Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"2","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405644756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405661285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405686712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:64779)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405846235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:58141)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405887281Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:51638)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.405921938Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.406634013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"2","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.40666238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"0","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.407216252Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"1","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.407358188Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"0","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.407383235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"2","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.407789053Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"1","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.407990102Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408017019Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[0]R flush_numbers[0..12]: [109, 114, 114, 112, 105, 111, 109, 112, 106, 112, 114, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408325207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[0]R generation[0..12]: [65, 67, 67, 66, 61, 66, 65, 66, 62, 66, 67, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408353594Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408369682Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[1]R flush_numbers[0..12]: [109, 114, 114, 112, 105, 111, 109, 112, 106, 112, 114, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408384451Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[1]R generation[0..12]: [65, 67, 67, 66, 61, 66, 65, 66, 62, 66, 67, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408397949Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408415087Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[2]R flush_numbers[0..12]: [109, 114, 114, 112, 105, 111, 109, 112, 106, 112, 114, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408429066Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[2]R generation[0..12]: [65, 67, 67, 66, 61, 66, 65, 66, 62, 66, 67, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408443755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408458013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408471162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.40848412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408496999Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408516557Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"":"downstairs","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408530086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408542024Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"0","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408555093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"1","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408568011Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"client":"2","":"downstairs","session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.4085817Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408593879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"e126ec39-3009-4d4b-ba3b-7f034568f589 is now active with session: d050313d-40a2-43b5-a59d-898bca1e3e55","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408606698Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408620656Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464,"session_id":"d050313d-40a2-43b5-a59d-898bca1e3e55"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408633015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} {"msg":"Activated sub_volume e126ec39-3009-4d4b-ba3b-7f034568f589","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:45.408643984Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 8192 data: 8 8 02/30 Read block 428 len 24576 03/30 Write block 313 len 20480 data: 8 6 7 6 5 04/30 Flush 05/30 Write block 392 len 24576 data: 10 8 8 5 5 4 06/30 Write block 540 len 36864 data: 5 6 6 4 5 6 7 6 6 07/30 Write block 92 len 40960 data: 4 3 2 3 3 3 3 3 3 4 08/30 Read block 329 len 12288 09/30 Read block 16 len 32768 10/30 Write block 260 len 40960 data: 5 5 3 4 5 6 5 5 6 6 11/30 Read block 490 len 12288 12/30 Read block 244 len 28672 13/30 Flush 14/30 Write block 381 len 20480 data: 12 12 11 10 8 15/30 Write block 196 len 28672 data: 8 6 5 7 6 6 6 16/30 Read block 530 len 40960 17/30 Read block 284 len 40960 18/30 Write block 61 len 8192 data: 9 9 19/30 Write block 152 len 16384 data: 6 7 8 7 20/30 Read block 203 len 8192 21/30 Read block 331 len 16384 22/30 Write block 55 len 24576 data: 8 6 7 9 10 9 23/30 Read block 546 len 16384 24/30 Read block 114 len 16384 25/30 Read block 223 len 24576 26/30 Write block 356 len 4096 data: 8 27/30 Write block 29 len 40960 data: 4 5 5 6 6 6 6 7 8 8 28/30 Write block 85 len 36864 data: 9 6 6 5 4 3 3 5 4 29/30 Read block 130 len 24576 30/30 Read block 295 len 4096 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 5 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 16 17 17 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:45.429619926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 67 67 67 114 114 114 T T T 2 040-059 67 67 67 114 114 114 T T T 3 060-079 66 66 66 112 112 112 T T T 4 080-099 68 68 68 116 116 116 T T T 5 100-119 68 68 68 116 116 116 F F F 6 120-139 65 65 65 109 109 109 F F F 7 140-159 66 66 66 112 112 112 T T T 8 160-179 62 62 62 106 106 106 F F F 9 180-199 68 68 68 115 115 115 T T T 10 200-219 67 67 67 114 114 114 T T T 11 220-239 66 66 66 112 112 112 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 65 65 65 109 109 109 F F F 15 300-319 68 68 68 115 115 115 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 65 65 65 109 109 109 T T T 18 360-379 66 66 66 112 112 112 F F F 19 380-399 68 68 68 116 116 116 T T T 20 400-419 66 66 66 113 113 113 F F F 21 420-439 67 67 67 114 114 114 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 66 66 66 112 112 112 F F F 25 500-519 65 65 65 109 109 109 F F F 26 520-539 65 65 65 109 109 109 F F F 27 540-559 68 68 68 116 116 116 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 68, Max flush: 116 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 69 69 69 117 117 117 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 69 69 69 117 117 117 F F F 4 080-099 69 69 69 117 117 117 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 65 65 65 109 109 109 F F F 7 140-159 69 69 69 117 117 117 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 69 69 69 117 117 117 F F F 10 200-219 69 69 69 117 117 117 F F F 11 220-239 66 66 66 112 112 112 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 68 68 68 116 116 116 F F F 14 280-299 65 65 65 109 109 109 F F F 15 300-319 68 68 68 115 115 115 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 69 69 69 117 117 117 F F F 18 360-379 66 66 66 112 112 112 F F F 19 380-399 69 69 69 117 117 117 F F F 20 400-419 66 66 66 113 113 113 F F F 21 420-439 67 67 67 114 114 114 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 66 66 66 112 112 112 F F F 25 500-519 65 65 65 109 109 109 F F F 26 520-539 65 65 65 109 109 109 F F F 27 540-559 68 68 68 116 116 116 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 69, Max flush: 117 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.686938235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.731584189Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.732219193Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.732255809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.732269828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.732284086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} {"msg":"Crucible 88c95d5c-960f-4c35-8130-647666f56a98 has session id: 0fdd9e8d-b0d2-4f27-8700-5bd0c2730099","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.732865786Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Upstairs opts: Upstairs UUID: 88c95d5c-960f-4c35-8130-647666f56a98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.732909721Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Crucible stats registered with UUID: 88c95d5c-960f-4c35-8130-647666f56a98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.732933069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:49.732953306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733369593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.73339696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} {"msg":"88c95d5c-960f-4c35-8130-647666f56a98 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733456044Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733474322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"0","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733502149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"0","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733516868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"1","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733535876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"1","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733549944Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733574802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733600649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733627076Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733653604Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"ds_connection connected from Ok(127.0.0.1:38257)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733678331Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"ds_connection connected from Ok(127.0.0.1:34095)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733705228Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:49.733732745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:49.733861562Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:49.733907567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.733934744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.734473638Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"0","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.734502345Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"1","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.735008792Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"1","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.73511976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"0","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.735621088Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:49.735708579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.736095504Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"ds_connection connected from Ok(127.0.0.1:59898)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.736241309Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.737213347Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.73776376Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[0]R flush_numbers[0..12]: [109, 117, 117, 117, 117, 116, 109, 117, 106, 117, 117, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738367247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[0]R generation[0..12]: [65, 69, 69, 69, 69, 68, 65, 69, 62, 69, 69, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738394024Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738412472Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[1]R flush_numbers[0..12]: [109, 117, 117, 117, 117, 116, 109, 117, 106, 117, 117, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.73842997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[1]R generation[0..12]: [65, 69, 69, 69, 69, 68, 65, 69, 62, 69, 69, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738446868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738464217Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[2]R flush_numbers[0..12]: [109, 117, 117, 117, 117, 116, 109, 117, 106, 117, 117, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738487824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[2]R generation[0..12]: [65, 69, 69, 69, 69, 68, 65, 69, 62, 69, 69, 66]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738505382Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738522651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738538479Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738557537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:02:59.738582644Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738606352Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"":"downstairs","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.73862254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738637239Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"0","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738653687Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"1","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738669945Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"client":"2","":"downstairs","session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738686254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738701102Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"88c95d5c-960f-4c35-8130-647666f56a98 is now active with session: 0fdd9e8d-b0d2-4f27-8700-5bd0c2730099","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.73871662Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738731179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472,"session_id":"0fdd9e8d-b0d2-4f27-8700-5bd0c2730099"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738749117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} {"msg":"Activated sub_volume 88c95d5c-960f-4c35-8130-647666f56a98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:02:59.738763286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 20 len 32768 data: 7 4 7 7 6 6 5 4 02/30 Read block 20 len 4096 03/30 Write block 88 len 32768 data: 6 5 4 4 6 5 3 4 04/30 Write block 396 len 36864 data: 6 5 8 9 7 7 8 9 8 05/30 Write block 545 len 12288 data: 7 8 7 06/30 Read block 531 len 28672 07/30 Read block 361 len 28672 08/30 Write block 537 len 8192 data: 6 6 09/30 Read block 402 len 8192 10/30 Write block 266 len 12288 data: 6 6 7 11/30 Write block 169 len 36864 data: 8 9 10 7 9 9 9 13 12 12/30 Write block 344 len 40960 data: 6 4 4 4 4 3 4 5 7 9 13/30 Write block 270 len 4096 data: 7 14/30 Flush 15/30 Write block 537 len 20480 data: 7 7 5 6 7 16/30 Write block 309 len 40960 data: 8 10 9 8 9 7 8 7 6 3 17/30 Flush 18/30 Write block 383 len 12288 data: 12 11 9 19/30 Read block 294 len 4096 20/30 Write block 168 len 24576 data: 7 9 10 11 8 10 21/30 Read block 208 len 16384 22/30 Read block 170 len 8192 23/30 Write block 231 len 12288 data: 9 7 6 24/30 Read block 201 len 32768 25/30 Write block 519 len 8192 data: 5 5 26/30 Write block 194 len 32768 data: 9 9 9 7 6 8 7 7 27/30 Read block 503 len 24576 28/30 Read block 529 len 8192 29/30 Read block 277 len 4096 30/30 Read block 203 len 28672 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:02:59.758883737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 69 69 69 117 117 117 F F F 4 080-099 70 70 70 118 118 118 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 65 65 65 109 109 109 F F F 7 140-159 69 69 69 117 117 117 F F F 8 160-179 70 70 70 118 118 118 T T T 9 180-199 69 69 69 117 117 117 T T T 10 200-219 69 69 69 117 117 117 T T T 11 220-239 66 66 66 112 112 112 T T T 12 240-259 66 66 66 112 112 112 F F F 13 260-279 70 70 70 118 118 118 F F F 14 280-299 65 65 65 109 109 109 F F F 15 300-319 70 70 70 119 119 119 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 70 70 70 118 118 118 F F F 18 360-379 66 66 66 112 112 112 F F F 19 380-399 70 70 70 118 118 118 T T T 20 400-419 70 70 70 118 118 118 F F F 21 420-439 67 67 67 114 114 114 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 66 66 66 112 112 112 F F F 25 500-519 65 65 65 109 109 109 T T T 26 520-539 70 70 70 119 119 119 T T T 27 540-559 70 70 70 119 119 119 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 70, Max flush: 119 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 69 69 69 117 117 117 F F F 4 080-099 70 70 70 118 118 118 F F F 5 100-119 68 68 68 116 116 116 F F F 6 120-139 65 65 65 109 109 109 F F F 7 140-159 69 69 69 117 117 117 F F F 8 160-179 71 71 71 120 120 120 F F F 9 180-199 71 71 71 120 120 120 F F F 10 200-219 71 71 71 120 120 120 F F F 11 220-239 71 71 71 120 120 120 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 70 70 70 118 118 118 F F F 14 280-299 65 65 65 109 109 109 F F F 15 300-319 70 70 70 119 119 119 F F F 16 320-339 62 62 62 106 106 106 F F F 17 340-359 70 70 70 118 118 118 F F F 18 360-379 66 66 66 112 112 112 F F F 19 380-399 71 71 71 120 120 120 F F F 20 400-419 70 70 70 118 118 118 F F F 21 420-439 67 67 67 114 114 114 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 66 66 66 112 112 112 F F F 24 480-499 66 66 66 112 112 112 F F F 25 500-519 71 71 71 120 120 120 F F F 26 520-539 71 71 71 120 120 120 F F F 27 540-559 70 70 70 119 119 119 F F F 28 560-579 66 66 66 112 112 112 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 71, Max flush: 120 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.013240273Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.058637329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.059328637Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.059364943Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.059383991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.059413248Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} {"msg":"Crucible 949f1292-876b-412b-b80b-7659c5f7dad7 has session id: 7d1ce12d-72f5-47fe-bc44-63b8c6bfc167","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060005876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Upstairs opts: Upstairs UUID: 949f1292-876b-412b-b80b-7659c5f7dad7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060039693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Crucible stats registered with UUID: 949f1292-876b-412b-b80b-7659c5f7dad7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060055431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:04.06006807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060620402Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.06064443Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} {"msg":"949f1292-876b-412b-b80b-7659c5f7dad7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060731711Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060749129Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"0","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060776716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"0","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060790725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"1","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060808793Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"1","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060873266Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"2","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060908112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"2","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.06093537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060963796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.060991264Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"ds_connection connected from Ok(127.0.0.1:56278)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.061019471Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"ds_connection connected from Ok(127.0.0.1:49356)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.061096733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"ds_connection connected from Ok(127.0.0.1:57982)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.061134959Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.06169741Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"0","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.061727727Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"1","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.061744275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"2","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.062357801Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"2","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.062386418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"0","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.062416475Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"1","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.062964158Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063069197Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[0]R flush_numbers[0..12]: [109, 118, 117, 117, 118, 116, 109, 117, 120, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063160827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[0]R generation[0..12]: [65, 70, 69, 69, 70, 68, 65, 69, 71, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063181755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063197384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[1]R flush_numbers[0..12]: [109, 118, 117, 117, 118, 116, 109, 117, 120, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063211802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[1]R generation[0..12]: [65, 70, 69, 69, 70, 68, 65, 69, 71, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063225881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063240059Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[2]R flush_numbers[0..12]: [109, 118, 117, 117, 118, 116, 109, 117, 120, 120, 120, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063254158Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[2]R generation[0..12]: [65, 70, 69, 69, 70, 68, 65, 69, 71, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063272136Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063297793Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.06332273Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063344308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Next flush: 121","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2025-03-18T05:03:04.063364456Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063378275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"":"downstairs","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063395653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063407952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"0","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.06342168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"1","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063434769Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"client":"2","":"downstairs","session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063447928Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063460246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"949f1292-876b-412b-b80b-7659c5f7dad7 is now active with session: 7d1ce12d-72f5-47fe-bc44-63b8c6bfc167","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063473135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063485124Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480,"session_id":"7d1ce12d-72f5-47fe-bc44-63b8c6bfc167"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063502502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} {"msg":"Activated sub_volume 949f1292-876b-412b-b80b-7659c5f7dad7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:04.063514541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 161 len 40960 data: 6 4 5 5 4 6 6 8 10 11 02/30 Write block 116 len 40960 data: 5 6 7 10 9 8 8 6 7 8 03/30 Read block 12 len 28672 04/30 Write block 281 len 12288 data: 6 7 9 05/30 Write block 85 len 40960 data: 10 7 7 7 6 5 5 7 6 4 06/30 Read block 361 len 24576 07/30 Read block 588 len 8192 08/30 Flush 09/30 Write block 358 len 4096 data: 8 10/30 Write block 570 len 24576 data: 6 6 7 6 7 7 11/30 Read block 30 len 28672 12/30 Write block 506 len 36864 data: 4 4 3 5 6 6 5 5 6 13/30 Read block 459 len 24576 14/30 Read block 273 len 32768 15/30 Read block 41 len 32768 16/30 Flush 17/30 Write block 320 len 12288 data: 3 4 5 18/30 Read block 565 len 24576 19/30 Write block 186 len 28672 data: 5 7 8 7 8 9 8 20/30 Flush 21/30 Write block 479 len 4096 data: 7 22/30 Read block 174 len 12288 23/30 Write block 488 len 32768 data: 8 8 8 5 6 5 5 5 24/30 Read block 119 len 16384 25/30 Read block 510 len 36864 26/30 Write block 211 len 32768 data: 6 5 8 8 7 6 6 6 27/30 Read block 82 len 4096 28/30 Write block 243 len 8192 data: 10 8 29/30 Write block 573 len 8192 data: 7 8 30/30 Write block 393 len 28672 data: 9 9 6 7 6 9 10 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:04.084703091Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 69 69 69 117 117 117 F F F 4 080-099 72 72 72 121 121 121 F F F 5 100-119 72 72 72 121 121 121 F F F 6 120-139 72 72 72 121 121 121 F F F 7 140-159 69 69 69 117 117 117 F F F 8 160-179 72 72 72 121 121 121 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 71 71 71 120 120 120 T T T 11 220-239 71 71 71 120 120 120 F F F 12 240-259 66 66 66 112 112 112 T T T 13 260-279 70 70 70 118 118 118 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 70 70 70 119 119 119 F F F 16 320-339 72 72 72 123 123 123 F F F 17 340-359 72 72 72 122 122 122 F F F 18 360-379 66 66 66 112 112 112 F F F 19 380-399 71 71 71 120 120 120 T T T 20 400-419 70 70 70 118 118 118 F F F 21 420-439 67 67 67 114 114 114 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 66 66 66 112 112 112 T T T 24 480-499 66 66 66 112 112 112 T T T 25 500-519 72 72 72 122 122 122 F F F 26 520-539 71 71 71 120 120 120 F F F 27 540-559 70 70 70 119 119 119 F F F 28 560-579 72 72 72 122 122 122 T T T 29 580-599 66 66 66 112 112 112 F F F Max gen: 72, Max flush: 123 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 69 69 69 117 117 117 F F F 4 080-099 72 72 72 121 121 121 F F F 5 100-119 72 72 72 121 121 121 F F F 6 120-139 72 72 72 121 121 121 F F F 7 140-159 69 69 69 117 117 117 F F F 8 160-179 72 72 72 121 121 121 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 73 73 73 124 124 124 F F F 11 220-239 71 71 71 120 120 120 F F F 12 240-259 73 73 73 124 124 124 F F F 13 260-279 70 70 70 118 118 118 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 70 70 70 119 119 119 F F F 16 320-339 72 72 72 123 123 123 F F F 17 340-359 72 72 72 122 122 122 F F F 18 360-379 66 66 66 112 112 112 F F F 19 380-399 73 73 73 124 124 124 F F F 20 400-419 70 70 70 118 118 118 F F F 21 420-439 67 67 67 114 114 114 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 72 72 72 122 122 122 F F F 26 520-539 71 71 71 120 120 120 F F F 27 540-559 70 70 70 119 119 119 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 73, Max flush: 124 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.329607614Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.376609063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.377290111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.377332367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.377351535Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.377369663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} {"msg":"Crucible 9574dc34-3712-4813-9f92-377c1af74c89 has session id: bf030530-be3e-496b-b0a4-abebff9f5f52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.377860582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Upstairs opts: Upstairs UUID: 9574dc34-3712-4813-9f92-377c1af74c89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.377895608Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Crucible stats registered with UUID: 9574dc34-3712-4813-9f92-377c1af74c89","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.377911507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:08.377924385Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378396426Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} {"msg":"9574dc34-3712-4813-9f92-377c1af74c89 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378451481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378478198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"0","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378508684Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"0","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378524313Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"1","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378539121Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"1","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378615553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"2","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.37864837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"2","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378825042Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378865137Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378894654Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:36134)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378922481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:46451)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378950149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:57494)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.378978555Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.379337138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"0","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.379384063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"1","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.379650096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"2","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.380143024Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"1","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.380234835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"0","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.380255472Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"2","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.380820034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.380923153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[0]R flush_numbers[0..12]: [109, 118, 117, 117, 121, 121, 121, 117, 121, 123, 124, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.380996775Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[0]R generation[0..12]: [65, 70, 69, 69, 72, 72, 72, 69, 72, 72, 73, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381015443Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381030261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[1]R flush_numbers[0..12]: [109, 118, 117, 117, 121, 121, 121, 117, 121, 123, 124, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.38104431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[1]R generation[0..12]: [65, 70, 69, 69, 72, 72, 72, 69, 72, 72, 73, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381057859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381071827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[2]R flush_numbers[0..12]: [109, 118, 117, 117, 121, 121, 121, 117, 121, 123, 124, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381091375Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[2]R generation[0..12]: [65, 70, 69, 69, 72, 72, 72, 69, 72, 72, 73, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381105754Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381119772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381132761Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381145689Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381158318Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:08.381171057Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"":"downstairs","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381189505Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381201364Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"0","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381214592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"1","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381227921Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"client":"2","":"downstairs","session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.38124111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381253478Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"9574dc34-3712-4813-9f92-377c1af74c89 is now active with session: bf030530-be3e-496b-b0a4-abebff9f5f52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381266117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381277906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488,"session_id":"bf030530-be3e-496b-b0a4-abebff9f5f52"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381289905Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} {"msg":"Activated sub_volume 9574dc34-3712-4813-9f92-377c1af74c89","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:08.381303013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 12288 data: 6 8 7 02/30 Read block 51 len 8192 03/30 Write block 339 len 20480 data: 4 4 5 8 7 04/30 Read block 379 len 40960 05/30 Write block 418 len 16384 data: 9 9 7 6 06/30 Flush 07/30 Write block 105 len 36864 data: 4 5 5 5 5 5 6 5 4 08/30 Write block 85 len 16384 data: 11 8 8 8 09/30 Write block 166 len 4096 data: 7 10/30 Read block 11 len 16384 11/30 Write block 309 len 24576 data: 9 11 10 9 10 8 12/30 Write block 379 len 12288 data: 7 10 13 13/30 Read block 371 len 8192 14/30 Write block 213 len 4096 data: 9 15/30 Read block 541 len 20480 16/30 Read block 231 len 8192 17/30 Write block 146 len 20480 data: 4 5 6 6 4 18/30 Read block 440 len 12288 19/30 Write block 419 len 8192 data: 10 8 20/30 Write block 535 len 36864 data: 4 6 8 8 6 7 8 7 5 21/30 Read block 310 len 24576 22/30 Read block 475 len 20480 23/30 Write block 384 len 16384 data: 12 10 9 6 24/30 Read block 75 len 40960 25/30 Flush 26/30 Write block 135 len 28672 data: 4 5 4 4 4 4 5 27/30 Read block 116 len 16384 28/30 Write block 435 len 12288 data: 7 7 8 29/30 Write block 171 len 4096 data: 12 30/30 Write block 546 len 8192 data: 9 8 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:08.399825952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 69 69 69 117 117 117 F F F 4 080-099 74 74 74 126 126 126 F F F 5 100-119 74 74 74 126 126 126 F F F 6 120-139 72 72 72 121 121 121 T T T 7 140-159 74 74 74 126 126 126 T T T 8 160-179 74 74 74 126 126 126 T T T 9 180-199 72 72 72 123 123 123 F F F 10 200-219 74 74 74 126 126 126 F F F 11 220-239 71 71 71 120 120 120 F F F 12 240-259 73 73 73 124 124 124 F F F 13 260-279 70 70 70 118 118 118 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 74 74 74 126 126 126 F F F 16 320-339 74 74 74 125 125 125 F F F 17 340-359 74 74 74 125 125 125 F F F 18 360-379 74 74 74 126 126 126 F F F 19 380-399 74 74 74 126 126 126 F F F 20 400-419 74 74 74 126 126 126 F F F 21 420-439 74 74 74 126 126 126 T T T 22 440-459 67 67 67 114 114 114 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 74 74 74 125 125 125 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 74 74 74 126 126 126 T T T 28 560-579 73 73 73 124 124 124 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 74, Max flush: 126 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 69 69 69 117 117 117 F F F 4 080-099 74 74 74 126 126 126 F F F 5 100-119 74 74 74 126 126 126 F F F 6 120-139 75 75 75 127 127 127 F F F 7 140-159 75 75 75 127 127 127 F F F 8 160-179 75 75 75 127 127 127 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 74 74 74 126 126 126 F F F 11 220-239 71 71 71 120 120 120 F F F 12 240-259 73 73 73 124 124 124 F F F 13 260-279 70 70 70 118 118 118 F F F 14 280-299 72 72 72 121 121 121 F F F 15 300-319 74 74 74 126 126 126 F F F 16 320-339 74 74 74 125 125 125 F F F 17 340-359 74 74 74 125 125 125 F F F 18 360-379 74 74 74 126 126 126 F F F 19 380-399 74 74 74 126 126 126 F F F 20 400-419 74 74 74 126 126 126 F F F 21 420-439 75 75 75 127 127 127 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 73 73 73 124 124 124 F F F 25 500-519 74 74 74 125 125 125 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 75 75 75 127 127 127 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 75, Max flush: 127 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.637648903Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.683945715Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.684671239Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.684721004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.684745572Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.684770579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} {"msg":"Crucible a2eeff52-7542-4243-9e86-31b7091a0121 has session id: 19e7b16f-86d0-423b-8b67-3fd54523f7bb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.685332841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Upstairs opts: Upstairs UUID: a2eeff52-7542-4243-9e86-31b7091a0121, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.685370337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Crucible stats registered with UUID: a2eeff52-7542-4243-9e86-31b7091a0121","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.685387575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:12.685400483Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.685882343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.68590807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} {"msg":"a2eeff52-7542-4243-9e86-31b7091a0121 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.685976293Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.685994852Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"0","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686025228Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"0","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686040757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"1","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686067514Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"1","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686093591Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"2","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686215118Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"2","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686261733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.68629715Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686323147Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:44269)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686350284Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59971)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686426716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55308)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.686521037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.68716234Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"0","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.687192547Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"2","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.687208015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"1","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.687812702Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"1","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.687942768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"0","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.687982324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"2","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688392761Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688513319Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[0]R flush_numbers[0..12]: [109, 118, 117, 117, 126, 126, 127, 127, 127, 123, 126, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688586731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[0]R generation[0..12]: [65, 70, 69, 69, 74, 74, 75, 75, 75, 72, 74, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688606189Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688624347Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[1]R flush_numbers[0..12]: [109, 118, 117, 117, 126, 126, 127, 127, 127, 123, 126, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688645085Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[1]R generation[0..12]: [65, 70, 69, 69, 74, 74, 75, 75, 75, 72, 74, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688670982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688698739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[2]R flush_numbers[0..12]: [109, 118, 117, 117, 126, 126, 127, 127, 127, 123, 126, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688717617Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[2]R generation[0..12]: [65, 70, 69, 69, 74, 74, 75, 75, 75, 72, 74, 71]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688738415Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688764512Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.68879086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688806818Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688820727Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:12.688843524Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"":"downstairs","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688872151Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688896489Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"0","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688911987Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"1","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688927926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"client":"2","":"downstairs","session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688950873Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688974191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"a2eeff52-7542-4243-9e86-31b7091a0121 is now active with session: 19e7b16f-86d0-423b-8b67-3fd54523f7bb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.688999168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.689019476Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496,"session_id":"19e7b16f-86d0-423b-8b67-3fd54523f7bb"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.689038344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} {"msg":"Activated sub_volume a2eeff52-7542-4243-9e86-31b7091a0121","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:12.689058642Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 485 len 8192 data: 6 6 02/30 Write block 423 len 24576 data: 4 5 7 7 8 8 03/30 Flush 04/30 Write block 77 len 28672 data: 6 6 8 6 9 9 10 05/30 Write block 112 len 32768 data: 6 5 4 6 6 7 8 11 06/30 Flush 07/30 Write block 281 len 24576 data: 7 8 10 11 8 8 08/30 Write block 341 len 4096 data: 6 09/30 Flush 10/30 Write block 318 len 4096 data: 4 11/30 Write block 237 len 36864 data: 7 7 8 6 10 10 11 9 7 12/30 Read block 505 len 24576 13/30 Write block 366 len 24576 data: 5 5 3 4 6 5 14/30 Read block 576 len 40960 15/30 Read block 574 len 24576 16/30 Read block 552 len 8192 17/30 Read block 138 len 36864 18/30 Read block 533 len 4096 19/30 Write block 234 len 4096 data: 5 20/30 Read block 232 len 4096 21/30 Write block 309 len 40960 data: 10 12 11 10 11 9 9 8 7 5 22/30 Write block 270 len 8192 data: 8 7 23/30 Read block 230 len 32768 24/30 Read block 55 len 12288 25/30 Write block 155 len 24576 data: 8 7 7 9 8 8 26/30 Read block 385 len 16384 27/30 Write block 255 len 28672 data: 4 5 4 3 5 6 6 28/30 Write block 391 len 4096 data: 7 29/30 Write block 280 len 32768 data: 7 8 9 11 12 9 9 10 30/30 Read block 268 len 36864 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 1 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 21 20 19 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:12.709149727Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 76 76 76 129 129 129 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 75 75 75 127 127 127 F F F 7 140-159 75 75 75 127 127 127 T T T 8 160-179 75 75 75 127 127 127 T T T 9 180-199 72 72 72 123 123 123 F F F 10 200-219 74 74 74 126 126 126 F F F 11 220-239 71 71 71 120 120 120 T T T 12 240-259 73 73 73 124 124 124 T T T 13 260-279 70 70 70 118 118 118 T T T 14 280-299 76 76 76 130 130 130 T T T 15 300-319 74 74 74 126 126 126 T T T 16 320-339 74 74 74 125 125 125 F F F 17 340-359 76 76 76 130 130 130 F F F 18 360-379 74 74 74 126 126 126 T T T 19 380-399 74 74 74 126 126 126 T T T 20 400-419 74 74 74 126 126 126 F F F 21 420-439 76 76 76 128 128 128 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 76 76 76 128 128 128 F F F 25 500-519 74 74 74 125 125 125 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 75 75 75 127 127 127 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 76, Max flush: 130 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 76 76 76 129 129 129 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 75 75 75 127 127 127 F F F 7 140-159 77 77 77 131 131 131 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 74 74 74 126 126 126 F F F 11 220-239 77 77 77 131 131 131 F F F 12 240-259 77 77 77 131 131 131 F F F 13 260-279 77 77 77 131 131 131 F F F 14 280-299 77 77 77 131 131 131 F F F 15 300-319 77 77 77 131 131 131 F F F 16 320-339 74 74 74 125 125 125 F F F 17 340-359 76 76 76 130 130 130 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 74 74 74 126 126 126 F F F 21 420-439 76 76 76 128 128 128 F F F 22 440-459 67 67 67 114 114 114 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 76 76 76 128 128 128 F F F 25 500-519 74 74 74 125 125 125 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 75 75 75 127 127 127 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 66 66 66 112 112 112 F F F Max gen: 77, Max flush: 131 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:16.97140157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.015517579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.016149933Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.01618521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.016200328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.016215967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} {"msg":"Crucible dd6a8552-2808-4779-805e-309d2e72acb1 has session id: 3444ba31-5353-439e-a6d8-9898cd42acfb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.016814084Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Upstairs opts: Upstairs UUID: dd6a8552-2808-4779-805e-309d2e72acb1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.016850451Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Crucible stats registered with UUID: dd6a8552-2808-4779-805e-309d2e72acb1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.016867119Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:17.016880097Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017394034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.01743075Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} {"msg":"dd6a8552-2808-4779-805e-309d2e72acb1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017488414Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017520031Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"0","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017566696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"0","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017596503Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"1","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.01762532Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"1","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017651637Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"2","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017678054Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"2","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017704782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017730339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017753936Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46938)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017779864Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:65271)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017806281Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36396)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.017833598Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.018549993Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"0","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.01858203Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"2","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.019222423Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"0","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.019252Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"2","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.019811542Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.019897773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.020142927Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"1","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.020650744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"1","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[0]R flush_numbers[0..12]: [109, 118, 117, 129, 129, 129, 127, 131, 131, 123, 126, 131]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021208206Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[0]R generation[0..12]: [65, 70, 69, 76, 76, 76, 75, 77, 77, 72, 74, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021233603Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021251841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[1]R flush_numbers[0..12]: [109, 118, 117, 129, 129, 129, 127, 131, 131, 123, 126, 131]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.02126914Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[1]R generation[0..12]: [65, 70, 69, 76, 76, 76, 75, 77, 77, 72, 74, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021287368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021304846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[2]R flush_numbers[0..12]: [109, 118, 117, 129, 129, 129, 127, 131, 131, 123, 126, 131]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021321884Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[2]R generation[0..12]: [65, 70, 69, 76, 76, 76, 75, 77, 77, 72, 74, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021342592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.02136022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Max found gen is 78","v":0,"nameWait for a query_work_queue command to finish before sending IO ":"crucible","level":30,"time":"2025-03-18T05:03:17.021383238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021399786Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021416354Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021432473Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"":"downstairs","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021448461Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021463179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"0","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021479668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"1","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021495766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"client":"2","":"downstairs","session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021512014Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021527013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"dd6a8552-2808-4779-805e-309d2e72acb1 is now active with session: 3444ba31-5353-439e-a6d8-9898cd42acfb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021542841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.02155772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504,"session_id":"3444ba31-5353-439e-a6d8-9898cd42acfb"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021572898Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} {"msg":"Activated sub_volume dd6a8552-2808-4779-805e-309d2e72acb1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:17.021586946Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 40960 data: 12 11 12 10 10 9 8 6 3 4 02/30 Write block 509 len 16384 data: 6 7 7 6 03/30 Flush 04/30 Write block 157 len 4096 data: 8 05/30 Read block 435 len 4096 06/30 Read block 145 len 12288 07/30 Write block 456 len 8192 data: 8 8 08/30 Write block 301 len 16384 data: 6 8 8 9 09/30 Read block 386 len 28672 10/30 Read block 443 len 20480 11/30 Write block 301 len 4096 data: 7 12/30 Flush 13/30 Write block 217 len 16384 data: 7 7 5 4 14/30 Read block 439 len 12288 15/30 Write block 298 len 36864 data: 8 9 7 8 9 9 10 8 6 16/30 Write block 429 len 16384 data: 8 8 9 10 17/30 Read block 165 len 24576 18/30 Write block 430 len 40960 data: 9 10 11 9 10 8 8 9 10 10 19/30 Read block 486 len 16384 20/30 Write block 211 len 36864 data: 7 6 10 9 8 7 8 8 6 21/30 Write block 135 len 24576 data: 5 6 5 5 5 5 22/30 Read block 240 len 12288 23/30 Read block 537 len 12288 24/30 Read block 233 len 20480 25/30 Write block 418 len 36864 data: 10 11 9 7 4 5 6 8 8 26/30 Write block 336 len 20480 data: 5 5 5 5 5 27/30 Read block 393 len 8192 28/30 Write block 257 len 36864 data: 5 4 6 7 7 4 5 6 7 29/30 Write block 90 len 12288 data: 6 6 8 30/30 Write block 593 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:17.040824531Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 76 76 76 129 129 129 T T T 5 100-119 76 76 76 129 129 129 F F F 6 120-139 75 75 75 127 127 127 T T T 7 140-159 78 78 78 133 133 133 T T T 8 160-179 77 77 77 131 131 131 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 74 74 74 126 126 126 T T T 11 220-239 77 77 77 131 131 131 T T T 12 240-259 77 77 77 131 131 131 T T T 13 260-279 77 77 77 131 131 131 T T T 14 280-299 77 77 77 131 131 131 T T T 15 300-319 78 78 78 133 133 133 T T T 16 320-339 78 78 78 132 132 132 T T T 17 340-359 76 76 76 130 130 130 T T T 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 74 74 74 126 126 126 T T T 21 420-439 76 76 76 128 128 128 T T T 22 440-459 78 78 78 133 133 133 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 76 76 76 128 128 128 F F F 25 500-519 78 78 78 132 132 132 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 75 75 75 127 127 127 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 66 66 66 112 112 112 T T T Max gen: 78, Max flush: 133 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 109 109 109 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 79 79 79 134 134 134 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 79 79 79 134 134 134 F F F 7 140-159 79 79 79 134 134 134 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 79 79 79 134 134 134 F F F 11 220-239 79 79 79 134 134 134 F F F 12 240-259 79 79 79 134 134 134 F F F 13 260-279 79 79 79 134 134 134 F F F 14 280-299 79 79 79 134 134 134 F F F 15 300-319 79 79 79 134 134 134 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 79 79 79 134 134 134 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 79 79 79 134 134 134 F F F 21 420-439 79 79 79 134 134 134 F F F 22 440-459 78 78 78 133 133 133 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 76 76 76 128 128 128 F F F 25 500-519 78 78 78 132 132 132 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 75 75 75 127 127 127 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 79 79 79 134 134 134 F F F Max gen: 79, Max flush: 134 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.328040108Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.373262152Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.373931302Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.373965669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.373984277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.373999065Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} {"msg":"Crucible a6e5d2bc-268c-437a-bff4-752cdbd8ef8c has session id: 6f3a18c1-5b51-4c96-b5f0-c456634e1b57","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.374571466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Upstairs opts: Upstairs UUID: a6e5d2bc-268c-437a-bff4-752cdbd8ef8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.374614491Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Crucible stats registered with UUID: a6e5d2bc-268c-437a-bff4-752cdbd8ef8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.374639509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:31.374661096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375181232Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375206689Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} {"msg":"a6e5d2bc-268c-437a-bff4-752cdbd8ef8c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.37530078Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375319598Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"0","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375347725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"0","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375361763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"1","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375375942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"1","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.37538976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"2","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375408248Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"2","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375421547Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375439215Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375503258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"ds_connection connected from Ok(127.0.0.1:61427)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375590589Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"ds_connection connected from Ok(127.0.0.1:39252)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375629805Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"ds_connection connected from Ok(127.0.0.1:42624)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.375812386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.376251911Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"2","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.376407114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"0","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.376776406Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"2","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.376889344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"0","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.377084034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"1","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.377259215Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.377395521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.377674142Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"1","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[0]R flush_numbers[0..12]: [109, 118, 117, 129, 134, 129, 134, 134, 131, 123, 134, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378184479Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[0]R generation[0..12]: [65, 70, 69, 76, 79, 76, 79, 79, 77, 72, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378204647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378220295Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[1]R flush_numbers[0..12]: [109, 118, 117, 129, 134, 129, 134, 134, 131, 123, 134, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378234304Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[1]R generation[0..12]: [65, 70, 69, 76, 79, 76, 79, 79, 77, 72, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378247982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378261931Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[2]R flush_numbers[0..12]: [109, 118, 117, 129, 134, 129, 134, 134, 131, 123, 134, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.37827566Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[2]R generation[0..12]: [65, 70, 69, 76, 79, 76, 79, 79, 77, 72, 79, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378289308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378303137Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378320315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:31.378344782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.37836966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378384048Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"":"downstairs","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378400127Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378412185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"0","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378426344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"1","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378439492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"client":"2","":"downstairs","session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378452601Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.3784646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"a6e5d2bc-268c-437a-bff4-752cdbd8ef8c is now active with session: 6f3a18c1-5b51-4c96-b5f0-c456634e1b57","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378477428Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378489317Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512,"session_id":"6f3a18c1-5b51-4c96-b5f0-c456634e1b57"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378501296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} {"msg":"Activated sub_volume a6e5d2bc-268c-437a-bff4-752cdbd8ef8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:31.378512235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 8192 data: 5 5 02/30 Write block 286 len 36864 data: 10 11 9 10 12 12 10 7 8 03/30 Read block 503 len 8192 04/30 Read block 412 len 12288 05/30 Write block 585 len 40960 data: 7 6 7 6 7 6 5 4 4 2 06/30 Read block 41 len 8192 07/30 Flush 08/30 Write block 232 len 16384 data: 8 7 6 6 09/30 Read block 467 len 32768 10/30 Read block 581 len 40960 11/30 Read block 508 len 40960 12/30 Read block 222 len 40960 13/30 Write block 285 len 32768 data: 10 11 12 10 11 13 13 11 14/30 Read block 547 len 32768 15/30 Read block 142 len 16384 16/30 Write block 83 len 28672 data: 11 10 12 9 9 9 7 17/30 Write block 280 len 4096 data: 8 18/30 Read block 352 len 4096 19/30 Write block 236 len 36864 data: 6 8 8 9 7 11 11 12 10 20/30 Read block 306 len 36864 21/30 Read block 362 len 36864 22/30 Write block 14 len 24576 data: 9 9 10 9 8 7 23/30 Read block 292 len 36864 24/30 Write block 13 len 28672 data: 8 10 10 11 10 9 8 25/30 Write block 355 len 20480 data: 9 9 9 9 6 26/30 Read block 436 len 24576 27/30 Read block 97 len 8192 28/30 Read block 541 len 8192 29/30 Read block 449 len 40960 30/30 Write block 505 len 32768 data: 4 5 5 4 7 8 8 7 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 4 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:31.400830387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 135 135 135 T T T 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 79 79 79 134 134 134 T T T 5 100-119 76 76 76 129 129 129 F F F 6 120-139 79 79 79 134 134 134 F F F 7 140-159 79 79 79 134 134 134 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 79 79 79 134 134 134 F F F 11 220-239 79 79 79 134 134 134 T T T 12 240-259 79 79 79 134 134 134 T T T 13 260-279 79 79 79 134 134 134 F F F 14 280-299 80 80 80 135 135 135 T T T 15 300-319 79 79 79 134 134 134 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 79 79 79 134 134 134 T T T 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 79 79 79 134 134 134 F F F 21 420-439 79 79 79 134 134 134 F F F 22 440-459 78 78 78 133 133 133 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 76 76 76 128 128 128 F F F 25 500-519 78 78 78 132 132 132 T T T 26 520-539 74 74 74 126 126 126 F F F 27 540-559 75 75 75 127 127 127 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 80 80 80 135 135 135 F F F Max gen: 80, Max flush: 135 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 136 136 136 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 79 79 79 134 134 134 F F F 7 140-159 79 79 79 134 134 134 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 79 79 79 134 134 134 F F F 11 220-239 81 81 81 136 136 136 F F F 12 240-259 81 81 81 136 136 136 F F F 13 260-279 79 79 79 134 134 134 F F F 14 280-299 81 81 81 136 136 136 F F F 15 300-319 79 79 79 134 134 134 F F F 16 320-339 79 79 79 134 134 134 F F F 17 340-359 81 81 81 136 136 136 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 79 79 79 134 134 134 F F F 21 420-439 79 79 79 134 134 134 F F F 22 440-459 78 78 78 133 133 133 F F F 23 460-479 73 73 73 124 124 124 F F F 24 480-499 76 76 76 128 128 128 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 75 75 75 127 127 127 F F F 28 560-579 73 73 73 124 124 124 F F F 29 580-599 80 80 80 135 135 135 F F F Max gen: 81, Max flush: 136 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.648069106Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.692781063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.693607737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.693643573Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.693657751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.69367244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} {"msg":"Crucible eec1619c-4092-47f5-b4a8-095b677a23d6 has session id: eef77751-68af-457d-8fca-f7d15104f8f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.694266258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Upstairs opts: Upstairs UUID: eec1619c-4092-47f5-b4a8-095b677a23d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.694305254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Crucible stats registered with UUID: eec1619c-4092-47f5-b4a8-095b677a23d6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.694327162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:35.69434042Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.694855736Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.694880894Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} {"msg":"eec1619c-4092-47f5-b4a8-095b677a23d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.694959536Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.694995362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"0","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695045357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"0","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695079223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"1","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695099061Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"1","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695124878Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"2","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695149006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"2","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695165764Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695183852Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695200651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36598)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695230487Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:38782)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695269343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60482)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.695303Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.696021085Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"0","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.696050952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"1","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.696232763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"2","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.696651869Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"0","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.696795844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"1","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697183454Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"2","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697249767Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697386463Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[0]R flush_numbers[0..12]: [136, 118, 117, 129, 136, 129, 134, 134, 131, 123, 134, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697781321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[0]R generation[0..12]: [81, 70, 69, 76, 81, 76, 79, 79, 77, 72, 79, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697801769Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697817198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[1]R flush_numbers[0..12]: [136, 118, 117, 129, 136, 129, 134, 134, 131, 123, 134, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697831416Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[1]R generation[0..12]: [81, 70, 69, 76, 81, 76, 79, 79, 77, 72, 79, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697849294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697863893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[2]R flush_numbers[0..12]: [136, 118, 117, 129, 136, 129, 134, 134, 131, 123, 134, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697877411Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[2]R generation[0..12]: [81, 70, 69, 76, 81, 76, 79, 79, 77, 72, 79, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.69789064Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697904458Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697917267Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697930096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697943125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:35.697956133Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"":"downstairs","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.697977401Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.6979894Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"0","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.698019066Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"1","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.698032845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"client":"2","":"downstairs","session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.698045784Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.698057842Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"eec1619c-4092-47f5-b4a8-095b677a23d6 is now active with session: eef77751-68af-457d-8fca-f7d15104f8f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.698072971Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.69808513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520,"session_id":"eef77751-68af-457d-8fca-f7d15104f8f6"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.698097188Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} {"msg":"Activated sub_volume eec1619c-4092-47f5-b4a8-095b677a23d6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:35.698108207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 24576 data: 7 8 7 7 8 7 02/30 Read block 95 len 8192 03/30 Write block 320 len 20480 data: 5 5 6 5 6 04/30 Flush 05/30 Write block 166 len 24576 data: 8 7 9 11 12 13 06/30 Write block 175 len 16384 data: 10 14 13 9 07/30 Read block 365 len 32768 08/30 Read block 200 len 16384 09/30 Read block 371 len 28672 10/30 Read block 172 len 24576 11/30 Flush 12/30 Flush 13/30 Flush 14/30 Write block 545 len 8192 data: 8 10 15/30 Write block 162 len 28672 data: 5 6 6 5 9 8 10 16/30 Read block 348 len 24576 17/30 Read block 364 len 8192 18/30 Read block 388 len 40960 19/30 Read block 449 len 12288 20/30 Write block 409 len 36864 data: 3 4 4 3 4 4 5 5 8 21/30 Read block 56 len 40960 22/30 Read block 483 len 40960 23/30 Flush 24/30 Flush 25/30 Write block 481 len 16384 data: 7 5 5 5 26/30 Read block 483 len 20480 27/30 Write block 467 len 16384 data: 3 4 4 4 28/30 Write block 557 len 32768 data: 6 5 4 5 5 5 6 6 29/30 Read block 428 len 20480 30/30 Read block 73 len 32768 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 5 5 6 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:35.719595576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 136 136 136 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 79 79 79 134 134 134 F F F 7 140-159 79 79 79 134 134 134 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 79 79 79 134 134 134 F F F 11 220-239 81 81 81 136 136 136 F F F 12 240-259 81 81 81 136 136 136 F F F 13 260-279 82 82 82 137 137 137 F F F 14 280-299 81 81 81 136 136 136 F F F 15 300-319 79 79 79 134 134 134 F F F 16 320-339 82 82 82 137 137 137 F F F 17 340-359 81 81 81 136 136 136 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 82 82 82 141 141 141 F F F 21 420-439 79 79 79 134 134 134 F F F 22 440-459 78 78 78 133 133 133 F F F 23 460-479 73 73 73 124 124 124 T T T 24 480-499 76 76 76 128 128 128 T T T 25 500-519 81 81 81 136 136 136 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 82 82 82 141 141 141 T T T 28 560-579 73 73 73 124 124 124 T T T 29 580-599 80 80 80 135 135 135 F F F Max gen: 82, Max flush: 141 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 136 136 136 F F F 1 020-039 70 70 70 118 118 118 F F F 2 040-059 69 69 69 117 117 117 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 81 81 81 136 136 136 F F F 5 100-119 76 76 76 129 129 129 F F F 6 120-139 79 79 79 134 134 134 F F F 7 140-159 79 79 79 134 134 134 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 79 79 79 134 134 134 F F F 11 220-239 81 81 81 136 136 136 F F F 12 240-259 81 81 81 136 136 136 F F F 13 260-279 82 82 82 137 137 137 F F F 14 280-299 81 81 81 136 136 136 F F F 15 300-319 79 79 79 134 134 134 F F F 16 320-339 82 82 82 137 137 137 F F F 17 340-359 81 81 81 136 136 136 F F F 18 360-379 77 77 77 131 131 131 F F F 19 380-399 77 77 77 131 131 131 F F F 20 400-419 82 82 82 141 141 141 F F F 21 420-439 79 79 79 134 134 134 F F F 22 440-459 78 78 78 133 133 133 F F F 23 460-479 83 83 83 142 142 142 F F F 24 480-499 83 83 83 142 142 142 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 83 83 83 142 142 142 F F F 28 560-579 83 83 83 142 142 142 F F F 29 580-599 80 80 80 135 135 135 F F F Max gen: 83, Max flush: 142 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.952028309Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.996406011Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.997055543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.997091769Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.997111287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.997126346Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} {"msg":"Crucible a715c09c-2b44-4ed4-b98f-a3554ad174a9 has session id: 5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.997686277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Upstairs opts: Upstairs UUID: a715c09c-2b44-4ed4-b98f-a3554ad174a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.997733082Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Crucible stats registered with UUID: a715c09c-2b44-4ed4-b98f-a3554ad174a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.99775776Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:39.997779207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998288034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998324341Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} {"msg":"a715c09c-2b44-4ed4-b98f-a3554ad174a9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998348088Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998376765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"0","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998405772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"0","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998420051Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"1","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998434669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"1","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998448048Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"2","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998465736Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"2","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998479904Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998493553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998532649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34019)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998561296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40533)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998609191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58533)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.998800821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.999349634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"2","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.99938318Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"0","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.999411907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"1","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:39.999995616Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"1","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000112724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"0","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000137701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"2","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000523791Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000697223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[0]R flush_numbers[0..12]: [136, 118, 117, 129, 136, 129, 134, 134, 141, 123, 134, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000787764Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[0]R generation[0..12]: [81, 70, 69, 76, 81, 76, 79, 79, 82, 72, 79, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000809551Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.00082474Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[1]R flush_numbers[0..12]: [136, 118, 117, 129, 136, 129, 134, 134, 141, 123, 134, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000838958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[1]R generation[0..12]: [81, 70, 69, 76, 81, 76, 79, 79, 82, 72, 79, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000852857Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000866876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[2]R flush_numbers[0..12]: [136, 118, 117, 129, 136, 129, 134, 134, 141, 123, 134, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000880924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[2]R generation[0..12]: [81, 70, 69, 76, 81, 76, 79, 79, 82, 72, 79, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000894813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000908971Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.00092171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000934788Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000947327Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000965995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"":"downstairs","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000979294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.000995412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"0","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001009301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"1","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001022469Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"client":"2","":"downstairs","session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001035668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001048106Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"a715c09c-2b44-4ed4-b98f-a3554ad174a9 is now active with session: 5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001060935Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001072794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528,"session_id":"5f52f988-2cb2-47ac-b4ce-47ab57d4ed9c"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001084773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} {"msg":"Activated sub_volume a715c09c-2b44-4ed4-b98f-a3554ad174a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:40.001096341Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 552 len 32768 data: 5 6 8 8 6 7 6 5 02/30 Write block 560 len 8192 data: 6 6 03/30 Read block 320 len 24576 04/30 Read block 347 len 4096 05/30 Flush 06/30 Write block 133 len 32768 data: 4 4 6 7 6 6 6 6 07/30 Write block 94 len 20480 data: 5 5 4 4 4 08/30 Read block 0 len 24576 09/30 Write block 276 len 24576 data: 5 6 5 6 9 9 10/30 Read block 558 len 28672 11/30 Flush 12/30 Write block 239 len 12288 data: 10 8 12 13/30 Write block 338 len 12288 data: 6 6 6 14/30 Read block 438 len 24576 15/30 Write block 38 len 28672 data: 9 8 8 8 7 8 8 16/30 Write block 559 len 12288 data: 6 7 7 17/30 Read block 464 len 36864 18/30 Read block 562 len 32768 19/30 Read block 97 len 24576 20/30 Read block 33 len 32768 21/30 Write block 578 len 40960 data: 5 6 5 6 4 4 6 8 7 8 22/30 Write block 251 len 20480 data: 4 5 4 5 5 23/30 Write block 357 len 8192 data: 10 10 24/30 Read block 321 len 32768 25/30 Write block 478 len 36864 data: 7 8 7 8 6 6 6 7 7 26/30 Write block 107 len 36864 data: 6 6 6 6 7 7 6 5 7 27/30 Write block 383 len 4096 data: 13 28/30 Read block 510 len 8192 29/30 Write block 353 len 32768 data: 10 8 10 10 11 11 7 6 30/30 Write block 438 len 12288 data: 11 11 9 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:40.020888997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 136 136 136 F F F 1 020-039 70 70 70 118 118 118 T T T 2 040-059 69 69 69 117 117 117 T T T 3 060-079 76 76 76 129 129 129 F F F 4 080-099 84 84 84 144 144 144 F F F 5 100-119 76 76 76 129 129 129 T T T 6 120-139 84 84 84 144 144 144 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 79 79 79 134 134 134 F F F 11 220-239 81 81 81 136 136 136 T T T 12 240-259 81 81 81 136 136 136 T T T 13 260-279 84 84 84 144 144 144 F F F 14 280-299 84 84 84 144 144 144 F F F 15 300-319 79 79 79 134 134 134 F F F 16 320-339 82 82 82 137 137 137 T T T 17 340-359 81 81 81 136 136 136 T T T 18 360-379 77 77 77 131 131 131 T T T 19 380-399 77 77 77 131 131 131 T T T 20 400-419 82 82 82 141 141 141 F F F 21 420-439 79 79 79 134 134 134 T T T 22 440-459 78 78 78 133 133 133 T T T 23 460-479 83 83 83 142 142 142 T T T 24 480-499 83 83 83 142 142 142 T T T 25 500-519 81 81 81 136 136 136 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 84 84 84 143 143 143 T T T 28 560-579 84 84 84 143 143 143 T T T 29 580-599 80 80 80 135 135 135 T T T Max gen: 84, Max flush: 144 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 136 136 136 F F F 1 020-039 85 85 85 145 145 145 F F F 2 040-059 85 85 85 145 145 145 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 84 84 84 144 144 144 F F F 5 100-119 85 85 85 145 145 145 F F F 6 120-139 84 84 84 144 144 144 F F F 7 140-159 84 84 84 144 144 144 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 79 79 79 134 134 134 F F F 11 220-239 85 85 85 145 145 145 F F F 12 240-259 85 85 85 145 145 145 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 84 84 84 144 144 144 F F F 15 300-319 79 79 79 134 134 134 F F F 16 320-339 85 85 85 145 145 145 F F F 17 340-359 85 85 85 145 145 145 F F F 18 360-379 85 85 85 145 145 145 F F F 19 380-399 85 85 85 145 145 145 F F F 20 400-419 82 82 82 141 141 141 F F F 21 420-439 85 85 85 145 145 145 F F F 22 440-459 85 85 85 145 145 145 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 85 85 85 145 145 145 F F F 28 560-579 85 85 85 145 145 145 F F F 29 580-599 85 85 85 145 145 145 F F F Max gen: 85, Max flush: 145 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.319297965Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:54.362235417Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.362607849Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.393862359Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.39442657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.394462007Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.394478175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.394495373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} {"msg":"Crucible 6fec20f5-a126-483d-8a20-b2d56a72a2c1 has session id: 8f022f91-d47e-464b-8521-f31538b8a091","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395246935Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Upstairs opts: Upstairs UUID: 6fec20f5-a126-483d-8a20-b2d56a72a2c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.39529593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Crucible stats registered with UUID: 6fec20f5-a126-483d-8a20-b2d56a72a2c1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395323387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:54.395338225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395680429Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395709237Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} {"msg":"6fec20f5-a126-483d-8a20-b2d56a72a2c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395750342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.39576646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"0","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395793618Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"0","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395807756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"1","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395822245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"1","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395835543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"2","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395849302Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"2","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.39587122Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395895207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395918505Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"ds_connection connected from Ok(127.0.0.1:47110)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.395943712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"ds_connection connected from Ok(127.0.0.1:60337)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.396148851Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"ds_connection connected from Ok(127.0.0.1:48784)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.396188607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.396648868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"2","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.396688364Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"1","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.396901522Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"0","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.397427907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"1","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.397455604Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"2","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.397470513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"0","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398020496Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398117415Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[0]R flush_numbers[0..12]: [136, 145, 145, 129, 144, 145, 144, 144, 141, 123, 134, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398191188Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[0]R generation[0..12]: [81, 85, 85, 76, 84, 85, 84, 84, 82, 72, 79, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398211176Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398226624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[1]R flush_numbers[0..12]: [136, 145, 145, 129, 144, 145, 144, 144, 141, 123, 134, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398240673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[1]R generation[0..12]: [81, 85, 85, 76, 84, 85, 84, 84, 82, 72, 79, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398257781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398272039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[2]R flush_numbers[0..12]: [136, 145, 145, 129, 144, 145, 144, 144, 141, 123, 134, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398285718Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[2]R generation[0..12]: [81, 85, 85, 76, 84, 85, 84, 84, 82, 72, 79, 85]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398299116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398313255Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398326324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398339312Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398351971Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.3983646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"":"downstairs","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398387547Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398399696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"0","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398412595Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"1","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398425393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"client":"2","":"downstairs","session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398438192Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398450191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"6fec20f5-a126-483d-8a20-b2d56a72a2c1 is now active with session: 8f022f91-d47e-464b-8521-f31538b8a091","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398464859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398476958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536,"session_id":"8f022f91-d47e-464b-8521-f31538b8a091"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398488937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} {"msg":"Activated sub_volume 6fec20f5-a126-483d-8a20-b2d56a72a2c1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:54.398500175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 437 len 20480 data: 10 12 12 10 8 02/30 Write block 1 len 28672 data: 5 6 6 6 6 8 8 03/30 Write block 33 len 40960 data: 7 7 7 8 9 10 9 9 9 8 04/30 Read block 59 len 20480 05/30 Read block 152 len 4096 06/30 Write block 288 len 36864 data: 11 12 14 14 12 8 9 7 7 07/30 Read block 170 len 4096 08/30 Write block 211 len 36864 data: 8 7 11 10 9 8 9 9 7 09/30 Read block 558 len 4096 10/30 Write block 231 len 12288 data: 10 9 8 11/30 Write block 573 len 28672 data: 8 9 8 6 5 6 7 12/30 Write block 316 len 28672 data: 10 9 7 4 6 6 7 13/30 Write block 408 len 8192 data: 2 4 14/30 Flush 15/30 Write block 135 len 36864 data: 7 8 7 7 7 7 6 4 5 16/30 Read block 282 len 16384 17/30 Write block 96 len 24576 data: 5 5 5 4 4 5 18/30 Read block 295 len 4096 19/30 Read block 129 len 20480 20/30 Write block 438 len 8192 data: 13 13 21/30 Flush 22/30 Flush 23/30 Write block 337 len 28672 data: 6 7 7 7 7 9 8 24/30 Flush 25/30 Write block 96 len 4096 data: 6 26/30 Read block 214 len 16384 27/30 Read block 151 len 36864 28/30 Read block 354 len 36864 29/30 Write block 547 len 16384 data: 9 7 6 5 30/30 Read block 213 len 12288 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:54.419053612Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 146 146 146 F F F 1 020-039 86 86 86 146 146 146 F F F 2 040-059 86 86 86 146 146 146 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 86 86 86 147 147 147 T T T 5 100-119 86 86 86 147 147 147 F F F 6 120-139 86 86 86 147 147 147 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 86 86 86 146 146 146 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 85 85 85 145 145 145 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 86 86 86 146 146 146 F F F 15 300-319 86 86 86 146 146 146 F F F 16 320-339 86 86 86 149 149 149 F F F 17 340-359 86 86 86 149 149 149 F F F 18 360-379 85 85 85 145 145 145 F F F 19 380-399 85 85 85 145 145 145 F F F 20 400-419 86 86 86 146 146 146 F F F 21 420-439 86 86 86 147 147 147 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 85 85 85 145 145 145 T T T 28 560-579 86 86 86 146 146 146 F F F 29 580-599 85 85 85 145 145 145 F F F Max gen: 86, Max flush: 149 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 146 146 146 F F F 1 020-039 86 86 86 146 146 146 F F F 2 040-059 86 86 86 146 146 146 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 87 87 87 150 150 150 F F F 5 100-119 86 86 86 147 147 147 F F F 6 120-139 86 86 86 147 147 147 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 86 86 86 146 146 146 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 85 85 85 145 145 145 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 86 86 86 146 146 146 F F F 15 300-319 86 86 86 146 146 146 F F F 16 320-339 86 86 86 149 149 149 F F F 17 340-359 86 86 86 149 149 149 F F F 18 360-379 85 85 85 145 145 145 F F F 19 380-399 85 85 85 145 145 145 F F F 20 400-419 86 86 86 146 146 146 F F F 21 420-439 86 86 86 147 147 147 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 85 85 85 145 145 145 F F F 25 500-519 81 81 81 136 136 136 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 86 86 86 146 146 146 F F F 29 580-599 85 85 85 145 145 145 F F F Max gen: 87, Max flush: 150 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.669758459Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.71405492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.7147158Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.714770815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.714796982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.7148147Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} {"msg":"Crucible ad6cd818-8113-4dd2-8f7d-402019a8a431 has session id: a146f27d-fad5-472c-a1ea-29537514c27b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.715360903Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Upstairs opts: Upstairs UUID: ad6cd818-8113-4dd2-8f7d-402019a8a431, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.715402419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Crucible stats registered with UUID: ad6cd818-8113-4dd2-8f7d-402019a8a431","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.715426646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:58.715447614Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.715959501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716007636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} {"msg":"ad6cd818-8113-4dd2-8f7d-402019a8a431 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716074419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716092467Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"0","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716121284Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"0","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716136032Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"1","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.71615946Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"1","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716189407Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"2","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716215414Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"2","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716238742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716264479Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716290566Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35096)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716381067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48239)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.716461978Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41310)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.71653876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.717205621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"2","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.717247607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"0","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.717738885Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"1","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.717826436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"0","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.717974291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"2","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.718477088Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"1","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.718522583Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.718580537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[0]R flush_numbers[0..12]: [146, 146, 146, 129, 150, 147, 147, 147, 141, 123, 146, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.7191308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[0]R generation[0..12]: [86, 86, 86, 76, 87, 86, 86, 86, 82, 72, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719151868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719166446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[1]R flush_numbers[0..12]: [146, 146, 146, 129, 150, 147, 147, 147, 141, 123, 146, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719180815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[1]R generation[0..12]: [86, 86, 86, 76, 87, 86, 86, 86, 82, 72, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719194544Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719208552Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[2]R flush_numbers[0..12]: [146, 146, 146, 129, 150, 147, 147, 147, 141, 123, 146, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719222501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[2]R generation[0..12]: [86, 86, 86, 76, 87, 86, 86, 86, 82, 72, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719236929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719250927Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719263756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719276655Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719289323Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719305192Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"":"downstairs","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719319Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719337849Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"0","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719351367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"1","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:03:58.719364466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"client":"2","":"downstairs","session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719383124Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719395372Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"ad6cd818-8113-4dd2-8f7d-402019a8a431 is now active with session: a146f27d-fad5-472c-a1ea-29537514c27b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719408271Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.71942015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544,"session_id":"a146f27d-fad5-472c-a1ea-29537514c27b"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719432359Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} {"msg":"Activated sub_volume ad6cd818-8113-4dd2-8f7d-402019a8a431","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:03:58.719443258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 487 len 20480 data: 8 9 9 9 6 02/30 Write block 508 len 36864 data: 5 8 9 9 8 6 7 5 4 03/30 Read block 18 len 16384 04/30 Write block 50 len 16384 data: 5 6 6 3 05/30 Write block 18 len 28672 data: 10 9 8 5 8 8 7 06/30 Read block 150 len 16384 07/30 Write block 427 len 4096 data: 9 08/30 Read block 522 len 12288 09/30 Write block 348 len 32768 data: 5 4 5 6 8 11 9 11 10/30 Read block 528 len 36864 11/30 Read block 19 len 12288 12/30 Write block 28 len 12288 data: 4 5 6 13/30 Write block 406 len 4096 data: 6 14/30 Read block 458 len 4096 15/30 Write block 94 len 40960 data: 6 6 7 6 6 5 5 6 6 4 16/30 Write block 201 len 24576 data: 8 7 5 7 7 7 17/30 Read block 317 len 12288 18/30 Read block 68 len 24576 19/30 Read block 60 len 4096 20/30 Read block 263 len 40960 21/30 Write block 399 len 32768 data: 11 8 8 9 10 9 7 7 22/30 Read block 364 len 40960 23/30 Read block 86 len 16384 24/30 Read block 551 len 4096 25/30 Write block 12 len 40960 data: 10 9 11 11 12 11 11 10 9 6 26/30 Flush 27/30 Write block 135 len 4096 data: 8 28/30 Write block 572 len 16384 data: 8 9 10 9 29/30 Read block 16 len 40960 30/30 Write block 248 len 8192 data: 5 6 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:03:58.74003229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 151 151 151 F F F 1 020-039 88 88 88 151 151 151 F F F 2 040-059 88 88 88 151 151 151 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 88 88 88 151 151 151 F F F 5 100-119 88 88 88 151 151 151 F F F 6 120-139 86 86 86 147 147 147 T T T 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 85 85 85 145 145 145 T T T 13 260-279 84 84 84 144 144 144 F F F 14 280-299 86 86 86 146 146 146 F F F 15 300-319 86 86 86 146 146 146 F F F 16 320-339 86 86 86 149 149 149 F F F 17 340-359 88 88 88 151 151 151 F F F 18 360-379 85 85 85 145 145 145 F F F 19 380-399 88 88 88 151 151 151 F F F 20 400-419 88 88 88 151 151 151 F F F 21 420-439 88 88 88 151 151 151 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 88 88 88 151 151 151 F F F 25 500-519 88 88 88 151 151 151 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 86 86 86 146 146 146 T T T 29 580-599 85 85 85 145 145 145 F F F Max gen: 88, Max flush: 151 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 151 151 151 F F F 1 020-039 88 88 88 151 151 151 F F F 2 040-059 88 88 88 151 151 151 F F F 3 060-079 76 76 76 129 129 129 F F F 4 080-099 88 88 88 151 151 151 F F F 5 100-119 88 88 88 151 151 151 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 72 72 72 123 123 123 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 89 89 89 152 152 152 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 86 86 86 146 146 146 F F F 15 300-319 86 86 86 146 146 146 F F F 16 320-339 86 86 86 149 149 149 F F F 17 340-359 88 88 88 151 151 151 F F F 18 360-379 85 85 85 145 145 145 F F F 19 380-399 88 88 88 151 151 151 F F F 20 400-419 88 88 88 151 151 151 F F F 21 420-439 88 88 88 151 151 151 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 88 88 88 151 151 151 F F F 25 500-519 88 88 88 151 151 151 F F F 26 520-539 74 74 74 126 126 126 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 89 89 89 152 152 152 F F F 29 580-599 85 85 85 145 145 145 F F F Max gen: 89, Max flush: 152 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:02.969515Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:03.012347763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.01275923Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044254645Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044288072Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044331347Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044363064Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044420698Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} {"msg":"Crucible 1759a728-8e21-4bce-b647-4ce5ab4545e2 has session id: 97ffb9ba-6530-447b-a3e6-04377701b701","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044852823Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Upstairs opts: Upstairs UUID: 1759a728-8e21-4bce-b647-4ce5ab4545e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044887869Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Crucible stats registered with UUID: 1759a728-8e21-4bce-b647-4ce5ab4545e2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.044911227Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:03.044929765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045350041Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045374598Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} {"msg":"1759a728-8e21-4bce-b647-4ce5ab4545e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045444041Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045461089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"0","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045488906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"0","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045508944Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"1","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045535532Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"1","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045562619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"2","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045591726Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"2","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045620533Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.04564586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045661248Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"ds_connection connected from Ok(127.0.0.1:42268)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045675757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"ds_connection connected from Ok(127.0.0.1:62880)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045696645Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"ds_connection connected from Ok(127.0.0.1:40849)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.045888735Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.046333169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"1","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.046462445Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"2","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.046652535Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"0","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.046965712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"1","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.046999069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"2","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047354162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"0","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047581228Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047675209Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[0]R flush_numbers[0..12]: [151, 151, 151, 129, 151, 151, 152, 147, 141, 123, 151, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047909744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[0]R generation[0..12]: [88, 88, 88, 76, 88, 88, 89, 86, 82, 72, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047931232Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.04794595Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[1]R flush_numbers[0..12]: [151, 151, 151, 129, 151, 151, 152, 147, 141, 123, 151, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047960009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[1]R generation[0..12]: [88, 88, 88, 76, 88, 88, 89, 86, 82, 72, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047973438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.047991765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[2]R flush_numbers[0..12]: [151, 151, 151, 129, 151, 151, 152, 147, 141, 123, 151, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048022092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[2]R generation[0..12]: [88, 88, 88, 76, 88, 88, 89, 86, 82, 72, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048036121Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048050019Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048062858Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048076237Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048089095Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":Wait for a query_work_queue command to finish before sending IO 1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048109073Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"":"downstairs","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048122252Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048134111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"0","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048147279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"1","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048160368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"client":"2","":"downstairs","session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048173417Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048185465Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"1759a728-8e21-4bce-b647-4ce5ab4545e2 is now active with session: 97ffb9ba-6530-447b-a3e6-04377701b701","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048198194Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048210063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552,"session_id":"97ffb9ba-6530-447b-a3e6-04377701b701"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.048224781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} {"msg":"Activated sub_volume 1759a728-8e21-4bce-b647-4ce5ab4545e2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:03.04823633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 184 len 32768 data: 6 5 6 8 9 8 9 10 02/30 Read block 420 len 8192 03/30 Write block 242 len 4096 data: 12 04/30 Write block 14 len 8192 data: 12 12 05/30 Write block 58 len 40960 data: 10 11 10 10 10 8 6 6 4 4 06/30 Write block 550 len 12288 data: 6 5 6 07/30 Write block 106 len 8192 data: 6 7 08/30 Read block 202 len 4096 09/30 Read block 249 len 36864 10/30 Write block 564 len 20480 data: 7 6 6 7 4 11/30 Write block 422 len 32768 data: 5 6 7 9 9 10 9 9 12/30 Read block 180 len 40960 13/30 Write block 409 len 32768 data: 5 5 5 4 5 5 6 6 14/30 Read block 331 len 12288 15/30 Flush 16/30 Write block 589 len 12288 data: 8 7 6 17/30 Write block 100 len 12288 data: 6 7 7 18/30 Write block 510 len 32768 data: 10 10 9 7 8 6 5 5 19/30 Read block 288 len 32768 20/30 Write block 521 len 36864 data: 4 6 7 7 7 7 6 7 6 21/30 Write block 87 len 32768 data: 10 10 8 7 7 9 7 7 22/30 Read block 470 len 4096 23/30 Write block 345 len 16384 data: 5 5 5 6 24/30 Write block 302 len 28672 data: 10 10 11 9 7 7 8 25/30 Write block 360 len 40960 data: 7 8 7 6 6 6 6 6 4 5 26/30 Write block 350 len 12288 data: 6 7 9 27/30 Write block 281 len 32768 data: 10 10 12 13 11 12 13 12 28/30 Read block 231 len 16384 29/30 Read block 268 len 20480 30/30 Read block 302 len 12288 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 15 15 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:03.06761028Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 153 153 153 F F F 1 020-039 88 88 88 151 151 151 F F F 2 040-059 90 90 90 153 153 153 F F F 3 060-079 90 90 90 153 153 153 F F F 4 080-099 88 88 88 151 151 151 T T T 5 100-119 90 90 90 153 153 153 T T T 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 90 90 90 153 153 153 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 90 90 90 153 153 153 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 86 86 86 146 146 146 T T T 15 300-319 86 86 86 146 146 146 T T T 16 320-339 86 86 86 149 149 149 F F F 17 340-359 88 88 88 151 151 151 T T T 18 360-379 85 85 85 145 145 145 T T T 19 380-399 88 88 88 151 151 151 F F F 20 400-419 90 90 90 153 153 153 F F F 21 420-439 90 90 90 153 153 153 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 88 88 88 151 151 151 F F F 25 500-519 88 88 88 151 151 151 T T T 26 520-539 74 74 74 126 126 126 T T T 27 540-559 90 90 90 153 153 153 F F F 28 560-579 90 90 90 153 153 153 F F F 29 580-599 85 85 85 145 145 145 T T T Max gen: 90, Max flush: 153 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 153 153 153 F F F 1 020-039 88 88 88 151 151 151 F F F 2 040-059 90 90 90 153 153 153 F F F 3 060-079 90 90 90 153 153 153 F F F 4 080-099 91 91 91 154 154 154 F F F 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 90 90 90 153 153 153 F F F 10 200-219 88 88 88 151 151 151 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 90 90 90 153 153 153 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 91 91 91 154 154 154 F F F 15 300-319 91 91 91 154 154 154 F F F 16 320-339 86 86 86 149 149 149 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 F F F 19 380-399 88 88 88 151 151 151 F F F 20 400-419 90 90 90 153 153 153 F F F 21 420-439 90 90 90 153 153 153 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 88 88 88 151 151 151 F F F 25 500-519 91 91 91 154 154 154 F F F 26 520-539 91 91 91 154 154 154 F F F 27 540-559 90 90 90 153 153 153 F F F 28 560-579 90 90 90 153 153 153 F F F 29 580-599 91 91 91 154 154 154 F F F Max gen: 91, Max flush: 154 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.355554284Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.399616509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.400201148Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.400234105Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.400248513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.400262852Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} {"msg":"Crucible c9e0f6ad-82ac-47f9-b505-6b95b2accb40 has session id: 9475c199-984f-4a30-ad4c-5b1380d2420c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.400769329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Upstairs opts: Upstairs UUID: c9e0f6ad-82ac-47f9-b505-6b95b2accb40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.400812944Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Crucible stats registered with UUID: c9e0f6ad-82ac-47f9-b505-6b95b2accb40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.400839591Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:07.400860689Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401346199Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401370256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} {"msg":"c9e0f6ad-82ac-47f9-b505-6b95b2accb40 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.4014271Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401454718Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"0","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401486584Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"0","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401502113Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"1","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401520041Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"1","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401533839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401548118Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401561416Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401584644Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401608371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52889)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401689833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45897)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401728639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:07.401780953Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:07.401858335Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:07.401897821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.401923309Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.402330706Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"1","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.402357613Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"0","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.402993357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"1","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.403021104Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"0","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.403599624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:07.403712682Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.402955755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50334)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.403175592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.404545809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405154126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[0]R flush_numbers[0..12]: [153, 151, 153, 153, 154, 154, 152, 147, 141, 153, 151, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405765152Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[0]R generation[0..12]: [90, 88, 90, 90, 91, 91, 89, 86, 82, 90, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405790339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405805747Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[1]R flush_numbers[0..12]: [153, 151, 153, 153, 154, 154, 152, 147, 141, 153, 151, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405819696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[1]R generation[0..12]: [90, 88, 90, 90, 91, 91, 89, 86, 82, 90, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405833455Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405847803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[2]R flush_numbers[0..12]: [153, 151, 153, 153, 154, 154, 152, 147, 141, 153, 151, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405867731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[2]R generation[0..12]: [90, 88, 90, 90, 91, 91, 89, 86, 82, 90, 88, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.40588197Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405896178Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405908907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405921886Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405934564Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405947233Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"":"downstairs","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405960162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.40597201Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"0","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405985149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"1","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.405998218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"client":"2","":"downstairs","session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.406011406Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.406023435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"c9e0f6ad-82ac-47f9-b505-6b95b2accb40 is now active with session: 9475c199-984f-4a30-ad4c-5b1380d2420c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.406036174Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"Set Active after no reconciliation","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-03-18T05:04:17.406057961Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560,"session_id":"9475c199-984f-4a30-ad4c-5b1380d2420c"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.40607144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} {"msg":"Activated sub_volume c9e0f6ad-82ac-47f9-b505-6b95b2accb40","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:17.406085298Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 32768 data: 5 6 7 6 7 8 8 8 02/30 Read block 440 len 12288 03/30 Flush 04/30 Write block 300 len 28672 data: 8 9 11 11 12 10 8 05/30 Write block 11 len 36864 data: 10 11 10 13 13 13 12 12 11 06/30 Read block 380 len 40960 07/30 Write block 535 len 24576 data: 5 7 9 9 7 8 08/30 Read block 144 len 8192 09/30 Write block 34 len 4096 data: 9 10/30 Read block 534 len 16384 11/30 Read block 583 len 4096 12/30 Flush 13/30 Write block 259 len 4096 data: 7 14/30 Write block 317 len 40960 data: 10 8 5 7 7 8 6 7 6 6 15/30 Read block 281 len 4096 16/30 Write block 26 len 40960 data: 6 5 6 7 8 7 8 9 10 9 17/30 Read block 82 len 4096 18/30 Read block 283 len 36864 19/30 Read block 298 len 8192 20/30 Read block 249 len 32768 21/30 Flush 22/30 Write block 91 len 28672 data: 8 10 8 8 7 8 7 23/30 Write block 83 len 40960 data: 12 11 13 10 11 11 9 8 9 11 24/30 Write block 219 len 4096 data: 8 25/30 Write block 301 len 32768 data: 10 12 12 13 11 9 8 9 26/30 Write block 498 len 36864 data: 5 5 5 7 9 8 7 5 6 27/30 Read block 212 len 4096 28/30 Read block 275 len 4096 29/30 Write block 382 len 20480 data: 13 14 13 11 10 30/30 Write block 566 len 32768 data: 7 8 5 5 7 7 9 10 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:17.426930755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 92 92 92 157 157 157 F F F 2 040-059 90 90 90 153 153 153 F F F 3 060-079 90 90 90 153 153 153 F F F 4 080-099 91 91 91 154 154 154 T T T 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 90 90 90 153 153 153 F F F 10 200-219 88 88 88 151 151 151 T T T 11 220-239 86 86 86 146 146 146 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 91 91 91 154 154 154 F F F 15 300-319 92 92 92 157 157 157 T T T 16 320-339 92 92 92 157 157 157 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 F F F 19 380-399 88 88 88 151 151 151 T T T 20 400-419 90 90 90 153 153 153 F F F 21 420-439 90 90 90 153 153 153 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 88 88 88 151 151 151 T T T 25 500-519 91 91 91 154 154 154 T T T 26 520-539 92 92 92 156 156 156 F F F 27 540-559 92 92 92 156 156 156 F F F 28 560-579 90 90 90 153 153 153 T T T 29 580-599 91 91 91 154 154 154 F F F Max gen: 92, Max flush: 157 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 92 92 92 157 157 157 F F F 2 040-059 90 90 90 153 153 153 F F F 3 060-079 90 90 90 153 153 153 F F F 4 080-099 93 93 93 158 158 158 F F F 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 90 90 90 153 153 153 F F F 10 200-219 93 93 93 158 158 158 F F F 11 220-239 86 86 86 146 146 146 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 91 91 91 154 154 154 F F F 15 300-319 93 93 93 158 158 158 F F F 16 320-339 92 92 92 157 157 157 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 F F F 19 380-399 93 93 93 158 158 158 F F F 20 400-419 90 90 90 153 153 153 F F F 21 420-439 90 90 90 153 153 153 F F F 22 440-459 86 86 86 146 146 146 F F F 23 460-479 85 85 85 145 145 145 F F F 24 480-499 93 93 93 158 158 158 F F F 25 500-519 93 93 93 158 158 158 F F F 26 520-539 92 92 92 156 156 156 F F F 27 540-559 92 92 92 156 156 156 F F F 28 560-579 93 93 93 158 158 158 F F F 29 580-599 91 91 91 154 154 154 F F F Max gen: 93, Max flush: 158 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.676728887Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:21.719927472Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.72032716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.751404479Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.751989888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.752025915Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.752044462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.752059321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} {"msg":"Crucible 7804373c-4c15-4595-b7b6-d779fd365d65 has session id: 9fb5275c-07cb-4918-8d20-7ac7f08c09f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.752671997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Upstairs opts: Upstairs UUID: 7804373c-4c15-4595-b7b6-d779fd365d65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.752715963Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Crucible stats registered with UUID: 7804373c-4c15-4595-b7b6-d779fd365d65","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.75274207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:21.752756918Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753177064Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753201092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} {"msg":"7804373c-4c15-4595-b7b6-d779fd365d65 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753272265Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753290792Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"0","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.75331914Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"0","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753333858Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"1","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753348657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"1","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753366835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"2","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753381703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"2","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753404301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753428208Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753451846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36027)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753536397Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61096)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753580592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40237)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.753607299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.754234054Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"1","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.754264651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"2","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.754429344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"0","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.754902335Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"1","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.754938311Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"2","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755023662Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"0","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755534209Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755559656Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[0]R flush_numbers[0..12]: [156, 157, 153, 153, 158, 154, 152, 147, 141, 153, 158, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755601122Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[0]R generation[0..12]: [92, 92, 90, 90, 93, 91, 89, 86, 82, 90, 93, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755639627Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[1]R flush_numbers[0..12]: [156, 157, 153, 153, 158, 154, 152, 147, 141, 153, 158, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755653906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[1]R generation[0..12]: [92, 92, 90, 90, 93, 91, 89, 86, 82, 90, 93, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755667625Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755681623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[2]R flush_numbers[0..12]: [156, 157, 153, 153, 158, 154, 152, 147, 141, 153, 158, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755695532Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[2]R generation[0..12]: [92, 92, 90, 90, 93, 91, 89, 86, 82, 90, 93, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.75570894Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755722779Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755735677Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755748936Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755761915Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755774873Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"":"downstairs","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755792062Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","Wait for a query_work_queue command to finish before sending IO pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.75581328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"0","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755826738Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"1","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755839837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"client":"2","":"downstairs","session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755852855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755865114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"7804373c-4c15-4595-b7b6-d779fd365d65 is now active with session: 9fb5275c-07cb-4918-8d20-7ac7f08c09f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755877933Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755889661Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568,"session_id":"9fb5275c-07cb-4918-8d20-7ac7f08c09f0"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.75590168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} {"msg":"Activated sub_volume 7804373c-4c15-4595-b7b6-d779fd365d65","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:21.755912889Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 444 len 40960 data: 7 9 8 8 6 8 8 7 8 8 02/30 Read block 88 len 12288 03/30 Read block 44 len 12288 04/30 Read block 358 len 8192 05/30 Read block 392 len 32768 06/30 Read block 48 len 32768 07/30 Read block 248 len 20480 08/30 Flush 09/30 Write block 290 len 40960 data: 15 15 13 9 10 8 8 8 9 10 10/30 Write block 485 len 8192 data: 8 8 11/30 Read block 480 len 24576 12/30 Read block 387 len 24576 13/30 Flush 14/30 Write block 218 len 24576 data: 10 9 5 3 7 5 15/30 Read block 107 len 8192 16/30 Read block 177 len 12288 17/30 Read block 221 len 16384 18/30 Read block 125 len 40960 19/30 Write block 389 len 40960 data: 4 6 8 11 10 10 7 8 7 10 20/30 Read block 307 len 12288 21/30 Write block 524 len 4096 data: 8 22/30 Read block 299 len 4096 23/30 Read block 336 len 4096 24/30 Write block 403 len 32768 data: 11 10 8 8 3 3 6 6 25/30 Write block 77 len 16384 data: 7 7 9 7 26/30 Read block 144 len 32768 27/30 Write block 464 len 4096 data: 4 28/30 Read block 379 len 36864 29/30 Write block 290 len 32768 data: 16 16 14 10 11 9 9 9 30/30 Write block 553 len 16384 data: 7 9 9 7 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:21.777208958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 92 92 92 157 157 157 F F F 2 040-059 90 90 90 153 153 153 F F F 3 060-079 90 90 90 153 153 153 T T T 4 080-099 93 93 93 158 158 158 T T T 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 90 90 90 153 153 153 F F F 10 200-219 93 93 93 158 158 158 T T T 11 220-239 86 86 86 146 146 146 T T T 12 240-259 92 92 92 157 157 157 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 94 94 94 160 160 160 T T T 15 300-319 93 93 93 158 158 158 F F F 16 320-339 92 92 92 157 157 157 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 F F F 19 380-399 93 93 93 158 158 158 T T T 20 400-419 90 90 90 153 153 153 T T T 21 420-439 90 90 90 153 153 153 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 85 85 85 145 145 145 T T T 24 480-499 94 94 94 160 160 160 F F F 25 500-519 93 93 93 158 158 158 F F F 26 520-539 92 92 92 156 156 156 T T T 27 540-559 92 92 92 156 156 156 T T T 28 560-579 93 93 93 158 158 158 F F F 29 580-599 91 91 91 154 154 154 F F F Max gen: 94, Max flush: 160 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 92 92 92 157 157 157 F F F 2 040-059 90 90 90 153 153 153 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 95 95 95 161 161 161 F F F 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 F F F 8 160-179 82 82 82 141 141 141 F F F 9 180-199 90 90 90 153 153 153 F F F 10 200-219 95 95 95 161 161 161 F F F 11 220-239 95 95 95 161 161 161 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 95 95 95 161 161 161 F F F 15 300-319 93 93 93 158 158 158 F F F 16 320-339 92 92 92 157 157 157 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 F F F 19 380-399 95 95 95 161 161 161 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 90 90 90 153 153 153 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 95 95 95 161 161 161 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 93 93 93 158 158 158 F F F 26 520-539 95 95 95 161 161 161 F F F 27 540-559 95 95 95 161 161 161 F F F 28 560-579 93 93 93 158 158 158 F F F 29 580-599 91 91 91 154 154 154 F F F Max gen: 95, Max flush: 161 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.068114882Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:26.110677264Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.111082492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.141947793Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.142606144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.142640491Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.142654769Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.142668888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} {"msg":"Crucible 8c4f3962-f3e4-4f2f-9012-22d1c2410ab3 has session id: 367c98e5-784f-49bd-b863-a2a195425692","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.14331963Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Upstairs opts: Upstairs UUID: 8c4f3962-f3e4-4f2f-9012-22d1c2410ab3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143357366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Crucible stats registered with UUID: 8c4f3962-f3e4-4f2f-9012-22d1c2410ab3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143378254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:26.143392872Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143758244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143791581Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} {"msg":"8c4f3962-f3e4-4f2f-9012-22d1c2410ab3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143845155Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143872582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"0","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143902449Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"0","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143917737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"1","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143932376Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"1","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143945605Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"2","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143960023Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"2","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.143984241Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.144008388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.144023057Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"ds_connection connected from Ok(127.0.0.1:36252)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.144036975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"ds_connection connected from Ok(127.0.0.1:50852)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.144060593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"ds_connection connected from Ok(127.0.0.1:37806)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.144152873Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.144777578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"1","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.144823343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"2","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.145013543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"0","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.145547618Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"1","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.145576135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"2","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.145698452Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"0","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146132637Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146258784Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[0]R flush_numbers[0..12]: [156, 157, 153, 161, 161, 154, 152, 147, 141, 153, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146329816Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[0]R generation[0..12]: [92, 92, 90, 95, 95, 91, 89, 86, 82, 90, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146351064Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146365802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[1]R flush_numbers[0..12]: [156, 157, 153, 161, 161, 154, 152, 147, 141, 153, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146379861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[1]R generation[0..12]: [92, 92, 90, 95, 95, 91, 89, 86, 82, 90, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146397429Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146412867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[2]R flush_numbers[0..12]: [156, 157, 153, 161, 161, 154, 152, 147, 141, 153, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146427116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[2]R generation[0..12]: [92, 92, 90, 95, 95, 91, 89, 86, 82, 90, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146440574Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146454453Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146467182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.14648015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146493119Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146505948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB",Wait for a query_work_queue command to finish before sending IO "pid":1576,"":"downstairs","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146525366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146537294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"0","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146550363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"1","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146563212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"client":"2","":"downstairs","session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.14657607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146590679Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"8c4f3962-f3e4-4f2f-9012-22d1c2410ab3 is now active with session: 367c98e5-784f-49bd-b863-a2a195425692","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146603557Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146615296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576,"session_id":"367c98e5-784f-49bd-b863-a2a195425692"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146627265Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} {"msg":"Activated sub_volume 8c4f3962-f3e4-4f2f-9012-22d1c2410ab3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:26.146638194Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 421 len 16384 data: 8 6 7 8 02/30 Read block 213 len 8192 03/30 Write block 390 len 8192 data: 7 9 04/30 Read block 262 len 12288 05/30 Read block 509 len 4096 06/30 Read block 526 len 8192 07/30 Write block 175 len 36864 data: 11 15 14 10 7 4 4 5 4 08/30 Read block 394 len 28672 09/30 Write block 464 len 4096 data: 5 10/30 Flush 11/30 Write block 572 len 32768 data: 10 11 11 10 7 6 7 8 12/30 Write block 39 len 36864 data: 10 10 10 9 9 9 5 5 5 13/30 Read block 578 len 12288 14/30 Read block 489 len 28672 15/30 Write block 462 len 32768 data: 5 4 6 4 3 4 5 5 16/30 Read block 252 len 24576 17/30 Read block 19 len 24576 18/30 Read block 481 len 16384 19/30 Read block 70 len 4096 20/30 Read block 136 len 12288 21/30 Read block 484 len 20480 22/30 Flush 23/30 Write block 572 len 28672 data: 11 12 12 11 8 7 8 24/30 Write block 80 len 4096 data: 8 25/30 Write block 515 len 40960 data: 7 6 6 5 6 6 5 7 8 9 26/30 Flush 27/30 Write block 144 len 28672 data: 5 5 5 6 7 7 5 28/30 Write block 159 len 40960 data: 9 9 7 6 7 7 6 10 9 11 29/30 Write block 530 len 4096 data: 5 30/30 Write block 158 len 28672 data: 10 10 10 8 7 8 8 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:26.16661563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 96 96 96 163 163 163 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 96 96 96 164 164 164 F F F 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 86 86 86 147 147 147 T T T 8 160-179 96 96 96 162 162 162 T T T 9 180-199 96 96 96 162 162 162 F F F 10 200-219 95 95 95 161 161 161 F F F 11 220-239 95 95 95 161 161 161 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 95 95 95 161 161 161 F F F 15 300-319 93 93 93 158 158 158 F F F 16 320-339 92 92 92 157 157 157 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 F F F 19 380-399 96 96 96 162 162 162 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 96 96 96 163 163 163 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 96 96 96 164 164 164 F F F 26 520-539 96 96 96 164 164 164 T T T 27 540-559 95 95 95 161 161 161 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 91 91 91 154 154 154 F F F Max gen: 96, Max flush: 164 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 96 96 96 163 163 163 F F F 2 040-059 96 96 96 163 163 163 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 96 96 96 164 164 164 F F F 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 F F F 7 140-159 97 97 97 165 165 165 F F F 8 160-179 97 97 97 165 165 165 F F F 9 180-199 96 96 96 162 162 162 F F F 10 200-219 95 95 95 161 161 161 F F F 11 220-239 95 95 95 161 161 161 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 84 84 84 144 144 144 F F F 14 280-299 95 95 95 161 161 161 F F F 15 300-319 93 93 93 158 158 158 F F F 16 320-339 92 92 92 157 157 157 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 F F F 19 380-399 96 96 96 162 162 162 F F F 20 400-419 95 95 95 161 161 161 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 96 96 96 163 163 163 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 96 96 96 164 164 164 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 95 95 95 161 161 161 F F F 28 560-579 96 96 96 164 164 164 F F F 29 580-599 91 91 91 154 154 154 F F F Max gen: 97, Max flush: 165 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.51099333Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.555973569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.55663293Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.556668487Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.556688674Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.556704223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} {"msg":"Crucible d8986e56-89bc-4b07-8149-c8357532bc6a has session id: deffbd53-aa9d-4ea4-a612-6557c6dfc602","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.557283852Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Upstairs opts: Upstairs UUID: d8986e56-89bc-4b07-8149-c8357532bc6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.557328418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Crucible stats registered with UUID: d8986e56-89bc-4b07-8149-c8357532bc6a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.557352805Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:30.557374643Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.557843884Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.55787981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} {"msg":"d8986e56-89bc-4b07-8149-c8357532bc6a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.557968681Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.557995208Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"0","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558026615Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"0","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558052802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"1","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558081239Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"1","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558116425Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"2","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558148092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"2","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558177399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558204086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558227554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"ds_connection connected from Ok(127.0.0.1:61954)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558277409Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"ds_connection connected from Ok(127.0.0.1:49339)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558310895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"ds_connection connected from Ok(127.0.0.1:34710)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.558390887Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.559021941Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"0","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.559052878Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"2","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.559339558Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"1","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.559887861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"0","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.559963603Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"2","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.55998703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"1","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560474279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560503407Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[0]R flush_numbers[0..12]: [156, 163, 163, 161, 164, 154, 152, 165, 165, 162, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560592507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[0]R generation[0..12]: [92, 96, 96, 95, 96, 91, 89, 97, 97, 96, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560613455Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560628443Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[1]R flush_numbers[0..12]: [156, 163, 163, 161, 164, 154, 152, 165, 165, 162, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560642412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[1]R generation[0..12]: [92, 96, 96, 95, 96, 91, 89, 97, 97, 96, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.5606559Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560670869Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[2]R flush_numbers[0..12]: [156, 163, 163, 161, 164, 154, 152, 165, 165, 162, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560684588Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[2]R generation[0..12]: [92, 96, 96, 95, 96, 91, 89, 97, 97, 96, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560698096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560712135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560732852Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:30.56075522Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560772168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560789937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"":"downstairs","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560804145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560824273Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"0","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560846821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"1","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560861699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"client":"2","":"downstairs","session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560874978Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560893256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"d8986e56-89bc-4b07-8149-c8357532bc6a is now active with session: deffbd53-aa9d-4ea4-a612-6557c6dfc602","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560915433Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560929502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584,"session_id":"deffbd53-aa9d-4ea4-a612-6557c6dfc602"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.560941871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} {"msg":"Activated sub_volume d8986e56-89bc-4b07-8149-c8357532bc6a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:30.56095295Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 205 len 28672 data: 8 8 7 6 6 5 9 02/30 Read block 82 len 40960 03/30 Read block 207 len 20480 04/30 Write block 301 len 16384 data: 11 13 13 14 05/30 Read block 330 len 8192 06/30 Read block 54 len 28672 07/30 Read block 64 len 12288 08/30 Write block 129 len 20480 data: 6 5 4 4 5 09/30 Read block 255 len 12288 10/30 Write block 220 len 8192 data: 6 4 11/30 Write block 554 len 24576 data: 10 10 8 8 7 7 12/30 Read block 562 len 24576 13/30 Write block 593 len 12288 data: 5 3 2 14/30 Write block 80 len 16384 data: 9 10 10 13 15/30 Read block 34 len 20480 16/30 Write block 177 len 28672 data: 15 11 8 5 5 6 5 17/30 Write block 364 len 40960 data: 7 7 7 7 5 6 7 6 6 4 18/30 Write block 207 len 36864 data: 8 7 7 6 10 8 12 11 10 19/30 Write block 187 len 24576 data: 9 10 9 10 11 9 20/30 Write block 556 len 20480 data: 9 9 8 8 8 21/30 Write block 403 len 40960 data: 12 11 9 9 4 4 7 7 6 5 22/30 Write block 507 len 8192 data: 6 6 23/30 Read block 84 len 4096 24/30 Read block 237 len 32768 25/30 Write block 277 len 8192 data: 7 6 26/30 Write block 51 len 36864 data: 7 7 4 6 9 7 8 11 12 27/30 Write block 325 len 36864 data: 7 7 7 7 8 6 7 6 4 28/30 Write block 36 len 24576 data: 9 10 11 11 11 11 29/30 Write block 181 len 32768 data: 6 7 6 7 6 7 10 11 30/30 Read block 392 len 36864 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 30 28 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:30.58003355Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 96 96 96 163 163 163 T T T 2 040-059 96 96 96 163 163 163 T T T 3 060-079 95 95 95 161 161 161 F F F 4 080-099 96 96 96 164 164 164 T T T 5 100-119 91 91 91 154 154 154 F F F 6 120-139 89 89 89 152 152 152 T T T 7 140-159 97 97 97 165 165 165 F F F 8 160-179 97 97 97 165 165 165 T T T 9 180-199 96 96 96 162 162 162 T T T 10 200-219 95 95 95 161 161 161 T T T 11 220-239 95 95 95 161 161 161 T T T 12 240-259 92 92 92 157 157 157 F F F 13 260-279 84 84 84 144 144 144 T T T 14 280-299 95 95 95 161 161 161 F F F 15 300-319 93 93 93 158 158 158 T T T 16 320-339 92 92 92 157 157 157 T T T 17 340-359 91 91 91 154 154 154 F F F 18 360-379 91 91 91 154 154 154 T T T 19 380-399 96 96 96 162 162 162 F F F 20 400-419 95 95 95 161 161 161 T T T 21 420-439 96 96 96 162 162 162 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 96 96 96 163 163 163 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 96 96 96 164 164 164 T T T 26 520-539 97 97 97 165 165 165 F F F 27 540-559 95 95 95 161 161 161 T T T 28 560-579 96 96 96 164 164 164 T T T 29 580-599 91 91 91 154 154 154 T T T Max gen: 97, Max flush: 165 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 98 98 98 166 166 166 F F F 2 040-059 98 98 98 166 166 166 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 98 98 98 166 166 166 F F F 5 100-119 91 91 91 154 154 154 F F F 6 120-139 98 98 98 166 166 166 F F F 7 140-159 97 97 97 165 165 165 F F F 8 160-179 98 98 98 166 166 166 F F F 9 180-199 98 98 98 166 166 166 F F F 10 200-219 98 98 98 166 166 166 F F F 11 220-239 98 98 98 166 166 166 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 98 98 98 166 166 166 F F F 14 280-299 95 95 95 161 161 161 F F F 15 300-319 98 98 98 166 166 166 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 98 98 98 166 166 166 F F F 19 380-399 96 96 96 162 162 162 F F F 20 400-419 98 98 98 166 166 166 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 94 94 94 159 159 159 F F F 23 460-479 96 96 96 163 163 163 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 98 98 98 166 166 166 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 98 98 98 166 166 166 F F F 29 580-599 98 98 98 166 166 166 F F F Max gen: 98, Max flush: 166 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.898808285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.942992317Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.943602803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.943638799Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.943653408Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.943667836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} {"msg":"Crucible d2a7ced7-32b1-404d-bb44-de29141c16ad has session id: 605ea3f2-51d3-430b-b85d-e11445e0d1b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.944280892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Upstairs opts: Upstairs UUID: d2a7ced7-32b1-404d-bb44-de29141c16ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.944325958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Crucible stats registered with UUID: d2a7ced7-32b1-404d-bb44-de29141c16ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.944351575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:44.944372333Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.944831245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.944862972Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} {"msg":"d2a7ced7-32b1-404d-bb44-de29141c16ad active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.944912306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.944931224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"0","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.94497327Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"0","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945033624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"1","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.94506843Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"1","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945093198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"2","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945120435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"2","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945145812Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.94517031Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945199967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33875)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945225804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:47853)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945252831Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43627)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945279798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.945806543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"0","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.94583701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"1","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.946473604Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"0","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.946621758Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"1","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.947115977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.947231865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.947310916Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"2","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.947821943Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"2","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[0]R flush_numbers[0..12]: [156, 166, 166, 161, 166, 154, 166, 165, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948348518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[0]R generation[0..12]: [92, 98, 98, 95, 98, 91, 98, 97, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948373176Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948392683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[1]R flush_numbers[0..12]: [156, 166, 166, 161, 166, 154, 166, 165, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948407302Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[1]R generation[0..12]: [92, 98, 98, 95, 98, 91, 98, 97, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948421091Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948434879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[2]R flush_numbers[0..12]: [156, 166, 166, 161, 166, 154, 166, 165, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948448288Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[2]R generation[0..12]: [92, 98, 98, 95, 98, 91, 98, 97, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948462006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948477245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Max found gen is 99","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948490224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Generation requested: 100 >= found:99","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948503192Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:04:44.948516031Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948531639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"":"downstairs","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948544758Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948556597Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"0","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948569685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"1","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948582584Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"client":"2","":"downstairs","session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948598012Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948610411Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"d2a7ced7-32b1-404d-bb44-de29141c16ad is now active with session: 605ea3f2-51d3-430b-b85d-e11445e0d1b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.94862311Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948634928Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592,"session_id":"605ea3f2-51d3-430b-b85d-e11445e0d1b0"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948647017Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} {"msg":"Activated sub_volume d2a7ced7-32b1-404d-bb44-de29141c16ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:44.948657986Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 279 len 8192 data: 7 10 03/30 Read block 398 len 36864 04/30 Read block 202 len 4096 05/30 Read block 66 len 36864 06/30 Read block 192 len 24576 07/30 Flush 08/30 Write block 477 len 12288 data: 8 8 9 09/30 Read block 226 len 8192 10/30 Read block 304 len 40960 11/30 Read block 425 len 28672 12/30 Flush 13/30 Write block 146 len 32768 data: 6 7 8 8 6 4 7 8 14/30 Write block 451 len 40960 data: 8 9 9 7 7 9 9 6 6 6 15/30 Write block 370 len 4096 data: 8 16/30 Write block 160 len 4096 data: 11 17/30 Write block 206 len 20480 data: 9 9 8 8 7 18/30 Read block 56 len 36864 19/30 Write block 451 len 24576 data: 9 10 10 8 8 10 20/30 Read block 513 len 4096 21/30 Read block 436 len 16384 22/30 Read block 113 len 40960 23/30 Write block 458 len 24576 data: 7 7 7 6 6 5 24/30 Read block 21 len 28672 25/30 Write block 97 len 36864 data: 8 7 6 7 8 8 5 4 5 26/30 Read block 144 len 24576 27/30 Write block 462 len 16384 data: 7 6 7 5 28/30 Write block 511 len 16384 data: 11 10 8 9 29/30 Write block 111 len 24576 data: 8 8 7 6 8 7 30/30 Read block 551 len 40960 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:44.969202404Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 98 98 98 166 166 166 F F F 2 040-059 98 98 98 166 166 166 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 98 98 98 166 166 166 T T T 5 100-119 91 91 91 154 154 154 T T T 6 120-139 98 98 98 166 166 166 F F F 7 140-159 97 97 97 165 165 165 T T T 8 160-179 98 98 98 166 166 166 T T T 9 180-199 98 98 98 166 166 166 F F F 10 200-219 98 98 98 166 166 166 T T T 11 220-239 98 98 98 166 166 166 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 100 100 100 168 168 168 F F F 14 280-299 100 100 100 168 168 168 F F F 15 300-319 98 98 98 166 166 166 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 98 98 98 166 166 166 T T T 19 380-399 96 96 96 162 162 162 F F F 20 400-419 98 98 98 166 166 166 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 94 94 94 159 159 159 T T T 23 460-479 100 100 100 169 169 169 T T T 24 480-499 94 94 94 160 160 160 F F F 25 500-519 98 98 98 166 166 166 T T T 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 98 98 98 166 166 166 F F F 29 580-599 98 98 98 166 166 166 F F F Max gen: 100, Max flush: 169 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 98 98 98 166 166 166 F F F 2 040-059 98 98 98 166 166 166 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 101 101 101 170 170 170 F F F 5 100-119 101 101 101 170 170 170 F F F 6 120-139 98 98 98 166 166 166 F F F 7 140-159 101 101 101 170 170 170 F F F 8 160-179 101 101 101 170 170 170 F F F 9 180-199 98 98 98 166 166 166 F F F 10 200-219 101 101 101 170 170 170 F F F 11 220-239 98 98 98 166 166 166 F F F 12 240-259 92 92 92 157 157 157 F F F 13 260-279 100 100 100 168 168 168 F F F 14 280-299 100 100 100 168 168 168 F F F 15 300-319 98 98 98 166 166 166 F F F 16 320-339 98 98 98 166 166 166 F F F 17 340-359 91 91 91 154 154 154 F F F 18 360-379 101 101 101 170 170 170 F F F 19 380-399 96 96 96 162 162 162 F F F 20 400-419 98 98 98 166 166 166 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 101 101 101 170 170 170 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 98 98 98 166 166 166 F F F 29 580-599 98 98 98 166 166 166 F F F Max gen: 101, Max flush: 170 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.228002797Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:49.271606179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.27198745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.30372709Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.304308029Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.304349255Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.304366713Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.304384601Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} {"msg":"Crucible 6f96991b-86f3-4398-bd67-37624757b3a7 has session id: e458a666-d33a-477e-b2c0-4a727f6c9dd8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305035324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Upstairs opts: Upstairs UUID: 6f96991b-86f3-4398-bd67-37624757b3a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.30506979Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Crucible stats registered with UUID: 6f96991b-86f3-4398-bd67-37624757b3a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305085628Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:49.305098567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305541261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305575357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} {"msg":"6f96991b-86f3-4398-bd67-37624757b3a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305630471Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305658559Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"0","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305690165Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"0","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305704684Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"1","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305718842Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"1","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305731881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"2","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305755398Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"2","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305780996Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305803893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305819442Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65325)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.30583347Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40703)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305847629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48782)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.305861367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.306635766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"1","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.306669433Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"0","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.306685821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"2","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.30727082Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"1","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.307308746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"2","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.307339813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"0","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.307821413Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.307924032Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[0]R flush_numbers[0..12]: [156, 166, 166, 161, 170, 170, 166, 170, 170, 166, 170, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.307996185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[0]R generation[0..12]: [92, 98, 98, 95, 101, 101, 98, 101, 101, 98, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308016582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308032171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[1]R flush_numbers[0..12]: [156, 166, 166, 161, 170, 170, 166, 170, 170, 166, 170, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308047799Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[1]R generation[0..12]: [92, 98, 98, 95, 101, 101, 98, 101, 101, 98, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308061678Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308076386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[2]R flush_numbers[0..12]: [156, 166, 166, 161, 170, 170, 166, 170, 170, 166, 170, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308090235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[2]R generation[0..12]: [92, 98, 98, 95, 101, 101, 98, 101, 101, 98, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308107943Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308123161Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.30813703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308150159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308162887Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308175576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"":"downstairs","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308188924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308200883Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"0","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308223461Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"1","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.30823665Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"client":"2","":"downstairs","session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308249648Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308262287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"6f96991b-86f3-4398-bd67-37624757b3a7 is now active with session: e458a666-d33a-477e-b2c0-4a727f6c9dd8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308274935Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308286654Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600,"session_id":"e458a666-d33a-477e-b2c0-4a727f6c9dd8"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308299263Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} {"msg":"Activated sub_volume 6f96991b-86f3-4398-bd67-37624757b3a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:49.308312372Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 157 len 24576 data: 9 11 11 12 9 8 02/30 Write block 295 len 36864 data: 10 10 10 10 11 9 12 14 14 03/30 Read block 188 len 40960 04/30 Read block 201 len 8192 05/30 Read block 458 len 40960 06/30 Read block 472 len 24576 07/30 Write block 200 len 12288 data: 8 9 8 08/30 Write block 317 len 28672 data: 11 9 6 8 8 9 7 09/30 Write block 344 len 28672 data: 7 6 6 6 7 5 7 10/30 Read block 187 len 8192 11/30 Write block 141 len 32768 data: 7 5 6 6 6 7 8 9 12/30 Read block 533 len 4096 13/30 Read block 74 len 20480 14/30 Write block 231 len 8192 data: 11 10 15/30 Write block 321 len 16384 data: 9 10 8 8 16/30 Flush 17/30 Write block 227 len 32768 data: 5 7 7 8 12 11 9 7 18/30 Write block 257 len 12288 data: 6 5 8 19/30 Write block 379 len 40960 data: 8 11 14 14 15 14 12 11 7 5 20/30 Read block 261 len 24576 21/30 Write block 113 len 40960 data: 8 7 9 8 8 9 12 10 9 9 22/30 Read block 574 len 36864 23/30 Read block 233 len 24576 24/30 Read block 23 len 36864 25/30 Write block 146 len 8192 data: 8 9 26/30 Write block 269 len 32768 data: 8 9 8 5 5 6 4 6 27/30 Read block 260 len 4096 28/30 Read block 562 len 4096 29/30 Read block 555 len 12288 30/30 Read block 426 len 32768 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 13 14 14 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:49.329407602Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 98 98 98 166 166 166 F F F 2 040-059 98 98 98 166 166 166 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 101 101 101 170 170 170 F F F 5 100-119 101 101 101 170 170 170 T T T 6 120-139 98 98 98 166 166 166 T T T 7 140-159 102 102 102 171 171 171 T T T 8 160-179 102 102 102 171 171 171 F F F 9 180-199 98 98 98 166 166 166 F F F 10 200-219 102 102 102 171 171 171 F F F 11 220-239 102 102 102 171 171 171 T T T 12 240-259 92 92 92 157 157 157 T T T 13 260-279 100 100 100 168 168 168 T T T 14 280-299 102 102 102 171 171 171 F F F 15 300-319 102 102 102 171 171 171 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 171 171 171 F F F 18 360-379 101 101 101 170 170 170 T T T 19 380-399 96 96 96 162 162 162 T T T 20 400-419 98 98 98 166 166 166 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 101 101 101 170 170 170 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 98 98 98 166 166 166 F F F 29 580-599 98 98 98 166 166 166 F F F Max gen: 102, Max flush: 171 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 98 98 98 166 166 166 F F F 2 040-059 98 98 98 166 166 166 F F F 3 060-079 95 95 95 161 161 161 F F F 4 080-099 101 101 101 170 170 170 F F F 5 100-119 103 103 103 172 172 172 F F F 6 120-139 103 103 103 172 172 172 F F F 7 140-159 103 103 103 172 172 172 F F F 8 160-179 102 102 102 171 171 171 F F F 9 180-199 98 98 98 166 166 166 F F F 10 200-219 102 102 102 171 171 171 F F F 11 220-239 103 103 103 172 172 172 F F F 12 240-259 103 103 103 172 172 172 F F F 13 260-279 103 103 103 172 172 172 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 102 102 102 171 171 171 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 171 171 171 F F F 18 360-379 103 103 103 172 172 172 F F F 19 380-399 103 103 103 172 172 172 F F F 20 400-419 98 98 98 166 166 166 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 101 101 101 170 170 170 F F F 23 460-479 101 101 101 170 170 170 F F F 24 480-499 94 94 94 160 160 160 F F F 25 500-519 101 101 101 170 170 170 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 98 98 98 166 166 166 F F F 29 580-599 98 98 98 166 166 166 F F F Max gen: 103, Max flush: 172 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.613139205Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:53.655729954Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.656102475Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.686808073Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.687393302Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.687439337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.687456485Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.687474173Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} {"msg":"Crucible df85a9d8-d874-4502-863f-efcbdc234882 has session id: b4bf3af3-724d-4eda-a2d4-ec244adadab4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688009157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Upstairs opts: Upstairs UUID: df85a9d8-d874-4502-863f-efcbdc234882, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688042404Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Crucible stats registered with UUID: df85a9d8-d874-4502-863f-efcbdc234882","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688057112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:53.688069761Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688457861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688481578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} {"msg":"df85a9d8-d874-4502-863f-efcbdc234882 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688546941Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.68856409Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"0","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688592297Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"0","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688606435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"1","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688620674Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"1","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688637432Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"2","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688660649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"2","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688683167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688706245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688729992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38720)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.68875498Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35688)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688780497Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42591)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.688807394Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.689572664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"0","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.689600782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"1","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.68961608Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"2","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690266322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"1","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690307798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"2","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690342074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"0","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690914694Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690950261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[0]R flush_numbers[0..12]: [156, 166, 166, 161, 170, 172, 172, 172, 171, 166, 171, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690968919Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[0]R generation[0..12]: [92, 98, 98, 95, 101, 103, 103, 103, 102, 98, 102, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690984147Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.690999086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[1]R flush_numbers[0..12]: [156, 166, 166, 161, 170, 172, 172, 172, 171, 166, 171, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691013454Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[1]R generation[0..12]: [92, 98, 98, 95, 101, 103, 103, 103, 102, 98, 102, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691027373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691041131Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[2]R flush_numbers[0..12]: [156, 166, 166, 161, 170, 172, 172, 172, 171, 166, 171, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.69105549Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[2]R generation[0..12]: [92, 98, 98, 95, 101, 103, 103, 103, 102, 98, 102, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691069268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691083077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691095995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691109054Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691121883Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691139431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"":"downstairs","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.69115328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691165088Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"0","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2025-03-18T05:04:53.691184956Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"1","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691198015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"client":"2","":"downstairs","session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691211134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691223292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"df85a9d8-d874-4502-863f-efcbdc234882 is now active with session: b4bf3af3-724d-4eda-a2d4-ec244adadab4","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691236371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.69124834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608,"session_id":"b4bf3af3-724d-4eda-a2d4-ec244adadab4"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691260328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} {"msg":"Activated sub_volume df85a9d8-d874-4502-863f-efcbdc234882","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:53.691271377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 171 len 12288 data: 14 9 11 02/30 Read block 81 len 8192 03/30 Read block 547 len 32768 04/30 Write block 458 len 4096 data: 8 05/30 Write block 592 len 32768 data: 5 6 4 3 2 2 2 2 06/30 Write block 29 len 36864 data: 8 9 8 9 10 11 10 10 11 07/30 Read block 336 len 20480 08/30 Read block 103 len 40960 09/30 Write block 510 len 12288 data: 11 12 11 10/30 Read block 496 len 28672 11/30 Read block 451 len 12288 12/30 Read block 22 len 16384 13/30 Read block 323 len 40960 14/30 Write block 561 len 40960 data: 8 6 7 8 7 8 9 6 6 8 15/30 Read block 117 len 28672 16/30 Write block 268 len 40960 data: 9 9 10 9 6 6 7 5 7 8 17/30 Read block 491 len 36864 18/30 Write block 445 len 12288 data: 10 9 9 19/30 Write block 266 len 16384 data: 8 8 10 10 20/30 Read block 13 len 20480 21/30 Write block 391 len 36864 data: 10 12 11 11 8 9 8 11 12 22/30 Read block 503 len 8192 23/30 Flush 24/30 Write block 471 len 40960 data: 4 5 5 5 6 7 9 9 10 8 25/30 Read block 100 len 4096 26/30 Write block 154 len 16384 data: 9 9 8 10 27/30 Write block 196 len 32768 data: 10 8 7 9 9 10 9 6 28/30 Read block 466 len 4096 29/30 Write block 55 len 24576 data: 10 8 9 12 13 11 30/30 Write block 565 len 24576 data: 8 9 10 7 7 9 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:53.712803012Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 104 104 104 173 173 173 F F F 2 040-059 98 98 98 166 166 166 T T T 3 060-079 95 95 95 161 161 161 T T T 4 080-099 101 101 101 170 170 170 F F F 5 100-119 103 103 103 172 172 172 F F F 6 120-139 103 103 103 172 172 172 F F F 7 140-159 103 103 103 172 172 172 T T T 8 160-179 104 104 104 173 173 173 F F F 9 180-199 98 98 98 166 166 166 T T T 10 200-219 102 102 102 171 171 171 T T T 11 220-239 103 103 103 172 172 172 F F F 12 240-259 103 103 103 172 172 172 F F F 13 260-279 104 104 104 173 173 173 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 102 102 102 171 171 171 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 171 171 171 F F F 18 360-379 103 103 103 172 172 172 F F F 19 380-399 104 104 104 173 173 173 F F F 20 400-419 98 98 98 166 166 166 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 104 104 104 173 173 173 F F F 23 460-479 101 101 101 170 170 170 T T T 24 480-499 94 94 94 160 160 160 T T T 25 500-519 104 104 104 173 173 173 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 104 104 104 173 173 173 T T T 29 580-599 104 104 104 173 173 173 F F F Max gen: 104, Max flush: 173 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 156 156 156 F F F 1 020-039 104 104 104 173 173 173 F F F 2 040-059 105 105 105 174 174 174 F F F 3 060-079 105 105 105 174 174 174 F F F 4 080-099 101 101 101 170 170 170 F F F 5 100-119 103 103 103 172 172 172 F F F 6 120-139 103 103 103 172 172 172 F F F 7 140-159 105 105 105 174 174 174 F F F 8 160-179 104 104 104 173 173 173 F F F 9 180-199 105 105 105 174 174 174 F F F 10 200-219 105 105 105 174 174 174 F F F 11 220-239 103 103 103 172 172 172 F F F 12 240-259 103 103 103 172 172 172 F F F 13 260-279 104 104 104 173 173 173 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 102 102 102 171 171 171 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 171 171 171 F F F 18 360-379 103 103 103 172 172 172 F F F 19 380-399 104 104 104 173 173 173 F F F 20 400-419 98 98 98 166 166 166 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 104 104 104 173 173 173 F F F 23 460-479 105 105 105 174 174 174 F F F 24 480-499 105 105 105 174 174 174 F F F 25 500-519 104 104 104 173 173 173 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 105 105 105 174 174 174 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 105, Max flush: 174 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:57.995175646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.039330072Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.040010301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.040053806Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.040073794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.040091602Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} {"msg":"Crucible d27a04a0-ef3e-4b7b-b685-020585be0aad has session id: 92e116ef-c264-439d-997e-2cdab02bf221","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.040821766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Upstairs opts: Upstairs UUID: d27a04a0-ef3e-4b7b-b685-020585be0aad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.040874991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Crucible stats registered with UUID: d27a04a0-ef3e-4b7b-b685-020585be0aad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.040905377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:58.040937274Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041380218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041407855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} {"msg":"d27a04a0-ef3e-4b7b-b685-020585be0aad active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041485577Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041504175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"0","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041534952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"0","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041560469Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"1","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041588896Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"1","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041615974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"2","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041678247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"2","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041712164Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041737271Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041761368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"ds_connection connected from Ok(127.0.0.1:41190)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041786926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"ds_connection connected from Ok(127.0.0.1:47959)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041913363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"ds_connection connected from Ok(127.0.0.1:61138)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.041957748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.042420799Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"2","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.042582983Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"0","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.043028156Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"2","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.043174731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"0","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.043618485Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.043796796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.044010194Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"1","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.044599912Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"1","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[0]R flush_numbers[0..12]: [156, 173, 174, 174, 170, 172, 172, 174, 173, 174, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045144175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[0]R generation[0..12]: [92, 104, 105, 105, 101, 103, 103, 105, 104, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045169193Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045184421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[1]R flush_numbers[0..12]: [156, 173, 174, 174, 170, 172, 172, 174, 173, 174, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.04519835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[1]R generation[0..12]: [92, 104, 105, 105, 101, 103, 103, 105, 104, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045215528Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045233586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[2]R flush_numbers[0..12]: [156, 173, 174, 174, 170, 172, 172, 174, 173, 174, 174, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045255254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[2]R generation[0..12]: [92, 104, 105, 105, 101, 103, 103, 105, 104, 105, 105, 103]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045272882Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.04529045Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045306558Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045330256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045346345Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045362423Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"":"downstairs","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045378831Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045393689Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"0","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045410388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"1","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045426966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"client":"2","":"downstairs","session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045443554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045458653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"d27a04a0-ef3e-4b7b-b685-020585be0aad is now active with session: 92e116ef-c264-439d-997e-2cdab02bf221","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045474421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.04548925Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616,"session_id":"92e116ef-c264-439d-997e-2cdab02bf221"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:04:58.045506808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} {"msg":"Activated sub_volume d27a04a0-ef3e-4b7b-b685-020585be0aad","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:04:58.045520976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 16384 data: 12 10 7 9 02/30 Write block 138 len 20480 data: 8 8 8 8 6 03/30 Read block 487 len 24576 04/30 Read block 175 len 32768 05/30 Write block 200 len 28672 data: 10 11 10 7 8 9 10 06/30 Flush 07/30 Write block 491 len 16384 data: 7 7 6 6 08/30 Write block 481 len 36864 data: 9 7 7 7 9 9 9 10 10 09/30 Write block 52 len 12288 data: 8 5 7 10/30 Read block 46 len 32768 11/30 Write block 383 len 4096 data: 16 12/30 Write block 73 len 28672 data: 4 6 7 7 8 8 10 13/30 Write block 375 len 16384 data: 5 4 6 7 14/30 Read block 521 len 28672 15/30 Read block 205 len 4096 16/30 Write block 214 len 36864 data: 12 11 9 10 11 10 7 5 8 17/30 Read block 356 len 28672 18/30 Write block 179 len 28672 data: 9 6 7 8 7 8 7 19/30 Read block 464 len 24576 20/30 Write block 372 len 12288 data: 7 5 4 21/30 Read block 225 len 32768 22/30 Flush 23/30 Write block 493 len 28672 data: 7 7 6 7 7 6 6 24/30 Flush 25/30 Write block 492 len 40960 data: 8 8 8 7 8 8 7 7 6 8 26/30 Write block 409 len 24576 data: 8 8 7 6 6 6 27/30 Read block 254 len 4096 28/30 Read block 1 len 20480 29/30 Write block 186 len 8192 data: 8 11 30/30 Read block 30 len 24576 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:04:58.066334126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 106 106 106 175 175 175 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 106 106 106 176 176 176 F F F 4 080-099 101 101 101 170 170 170 F F F 5 100-119 103 103 103 172 172 172 F F F 6 120-139 106 106 106 175 175 175 F F F 7 140-159 106 106 106 175 175 175 F F F 8 160-179 106 106 106 176 176 176 F F F 9 180-199 106 106 106 176 176 176 T T T 10 200-219 106 106 106 176 176 176 F F F 11 220-239 106 106 106 176 176 176 F F F 12 240-259 103 103 103 172 172 172 F F F 13 260-279 104 104 104 173 173 173 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 102 102 102 171 171 171 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 171 171 171 F F F 18 360-379 106 106 106 176 176 176 F F F 19 380-399 106 106 106 176 176 176 F F F 20 400-419 98 98 98 166 166 166 T T T 21 420-439 96 96 96 162 162 162 F F F 22 440-459 104 104 104 173 173 173 F F F 23 460-479 105 105 105 174 174 174 F F F 24 480-499 106 106 106 177 177 177 T T T 25 500-519 104 104 104 173 173 173 T T T 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 105 105 105 174 174 174 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 106, Max flush: 177 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 106 106 106 175 175 175 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 106 106 106 176 176 176 F F F 4 080-099 101 101 101 170 170 170 F F F 5 100-119 103 103 103 172 172 172 F F F 6 120-139 106 106 106 175 175 175 F F F 7 140-159 106 106 106 175 175 175 F F F 8 160-179 106 106 106 176 176 176 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 106 106 106 176 176 176 F F F 12 240-259 103 103 103 172 172 172 F F F 13 260-279 104 104 104 173 173 173 F F F 14 280-299 102 102 102 171 171 171 F F F 15 300-319 102 102 102 171 171 171 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 171 171 171 F F F 18 360-379 106 106 106 176 176 176 F F F 19 380-399 106 106 106 176 176 176 F F F 20 400-419 107 107 107 178 178 178 F F F 21 420-439 96 96 96 162 162 162 F F F 22 440-459 104 104 104 173 173 173 F F F 23 460-479 105 105 105 174 174 174 F F F 24 480-499 107 107 107 178 178 178 F F F 25 500-519 107 107 107 178 178 178 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 105 105 105 174 174 174 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 107, Max flush: 178 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.300795716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:02.345356649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.345757388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.376620879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.377200989Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.377236675Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.377255833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.377270831Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} {"msg":"Crucible 0f59751e-4f28-438a-bdc9-fbb903f157ea has session id: 6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.377847741Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Upstairs opts: Upstairs UUID: 0f59751e-4f28-438a-bdc9-fbb903f157ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.377892167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Crucible stats registered with UUID: 0f59751e-4f28-438a-bdc9-fbb903f157ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.377916534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:02.377938462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378285686Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378310953Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} {"msg":"0f59751e-4f28-438a-bdc9-fbb903f157ea active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378376346Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378394844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"0","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378422591Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"0","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.37843628Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"1","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378450908Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"1","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378468596Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"2","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378482945Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"2","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378495824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378509362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378522781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"ds_connection connected from Ok(127.0.0.1:43992)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378609352Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"ds_connection connected from Ok(127.0.0.1:33919)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378647748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"ds_connection connected from Ok(127.0.0.1:51557)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.378677525Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.379360403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"1","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.37938924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"2","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.379412378Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"0","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380081528Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"1","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380113185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"2","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380130473Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"0","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380725961Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380751098Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[0]R flush_numbers[0..12]: [175, 175, 176, 176, 170, 172, 175, 175, 176, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380793214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[0]R generation[0..12]: [106, 106, 106, 106, 101, 103, 106, 106, 106, 107, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380810232Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380824681Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[1]R flush_numbers[0..12]: [175, 175, 176, 176, 170, 172, 175, 175, 176, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380838829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[1]R generation[0..12]: [106, 106, 106, 106, 101, 103, 106, 106, 106, 107, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380852728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380866696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[2]R flush_numbers[0..12]: [175, 175, 176, 176, 170, 172, 175, 175, 176, 178, 176, 176]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380880215Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[2]R generation[0..12]: [106, 106, 106, 106, 101, 103, 106, 106, 106, 107, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380893703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380907602Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380920431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380939129Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:02.380962456Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.380982974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"":"downstairs","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381000352Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.38102223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"0","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381046938Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"1","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381061986Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"client":"2","":"downstairs","session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381075504Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381087783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"0f59751e-4f28-438a-bdc9-fbb903f157ea is now active with session: 6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381109311Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381128619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624,"session_id":"6603c5c8-fc09-4f3e-9c8e-7f4db9ab9b75"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381141518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} {"msg":"Activated sub_volume 0f59751e-4f28-438a-bdc9-fbb903f157ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:02.381153416Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 375 len 4096 data: 6 03/30 Read block 142 len 16384 04/30 Write block 293 len 32768 data: 11 12 11 11 11 11 12 10 05/30 Flush 06/30 Flush 07/30 Write block 357 len 4096 data: 12 08/30 Write block 394 len 28672 data: 12 9 10 9 12 13 9 09/30 Write block 452 len 24576 data: 11 11 9 9 11 10 10/30 Write block 312 len 4096 data: 12 11/30 Read block 587 len 20480 12/30 Write block 239 len 16384 data: 11 9 13 13 13/30 Write block 427 len 12288 data: 11 10 10 14/30 Read block 101 len 40960 15/30 Read block 488 len 32768 16/30 Read block 87 len 16384 17/30 Read block 587 len 32768 18/30 Read block 174 len 4096 19/30 Read block 495 len 28672 20/30 Write block 90 len 36864 data: 9 10 12 9 9 8 9 9 8 21/30 Read block 175 len 36864 22/30 Write block 574 len 24576 data: 13 12 9 8 9 9 23/30 Read block 541 len 36864 24/30 Write block 229 len 32768 data: 8 9 13 12 10 8 7 7 25/30 Read block 336 len 8192 26/30 Read block 597 len 8192 27/30 Read block 382 len 36864 28/30 Read block 234 len 40960 29/30 Write block 123 len 20480 data: 7 8 9 7 8 30/30 Write block 410 len 24576 data: 9 8 7 7 7 7 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:02.400994517Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 106 106 106 175 175 175 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 106 106 106 176 176 176 F F F 4 080-099 101 101 101 170 170 170 T T T 5 100-119 103 103 103 172 172 172 F F F 6 120-139 106 106 106 175 175 175 T T T 7 140-159 106 106 106 175 175 175 F F F 8 160-179 106 106 106 176 176 176 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 106 106 106 176 176 176 T T T 12 240-259 103 103 103 172 172 172 T T T 13 260-279 104 104 104 173 173 173 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 108 108 108 180 180 180 T T T 16 320-339 102 102 102 171 171 171 F F F 17 340-359 102 102 102 171 171 171 T T T 18 360-379 108 108 108 180 180 180 F F F 19 380-399 106 106 106 176 176 176 T T T 20 400-419 107 107 107 178 178 178 T T T 21 420-439 96 96 96 162 162 162 T T T 22 440-459 104 104 104 173 173 173 T T T 23 460-479 105 105 105 174 174 174 F F F 24 480-499 107 107 107 178 178 178 F F F 25 500-519 107 107 107 178 178 178 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 105 105 105 174 174 174 T T T 29 580-599 104 104 104 173 173 173 F F F Max gen: 108, Max flush: 180 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 106 106 106 175 175 175 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 106 106 106 176 176 176 F F F 4 080-099 109 109 109 181 181 181 F F F 5 100-119 103 103 103 172 172 172 F F F 6 120-139 109 109 109 181 181 181 F F F 7 140-159 106 106 106 175 175 175 F F F 8 160-179 106 106 106 176 176 176 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 109 109 109 181 181 181 F F F 12 240-259 109 109 109 181 181 181 F F F 13 260-279 104 104 104 173 173 173 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 109 109 109 181 181 181 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 109 109 109 181 181 181 F F F 18 360-379 108 108 108 180 180 180 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 109 109 109 181 181 181 F F F 21 420-439 109 109 109 181 181 181 F F F 22 440-459 109 109 109 181 181 181 F F F 23 460-479 105 105 105 174 174 174 F F F 24 480-499 107 107 107 178 178 178 F F F 25 500-519 107 107 107 178 178 178 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 109 109 109 181 181 181 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 109, Max flush: 181 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.703789393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.748039778Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.748636806Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.748672592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.74868773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.748703699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} {"msg":"Crucible d61be264-b735-4126-98d0-81a8e2073260 has session id: e98e2dd6-db12-43ff-b02e-701619b87658","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.74926902Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Upstairs opts: Upstairs UUID: d61be264-b735-4126-98d0-81a8e2073260, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749314365Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Crucible stats registered with UUID: d61be264-b735-4126-98d0-81a8e2073260","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749343072Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:06.74936438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749810564Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749837361Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} {"msg":"d61be264-b735-4126-98d0-81a8e2073260 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749901144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749919752Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"0","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749950519Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"0","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749965257Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"1","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749979686Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"1","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.749993534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"2","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750008123Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"2","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750027261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750054388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750082795Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"ds_connection connected from Ok(127.0.0.1:56308)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750165396Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"ds_connection connected from Ok(127.0.0.1:60408)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750210692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"ds_connection connected from Ok(127.0.0.1:34994)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750239349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.75090363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"2","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.750943115Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"0","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.751533514Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"0","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.751627154Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"2","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.751721334Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"1","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.752090866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.75224422Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.75243179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"1","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[0]R flush_numbers[0..12]: [175, 175, 176, 176, 181, 172, 181, 175, 176, 178, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.752945017Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[0]R generation[0..12]: [106, 106, 106, 106, 109, 103, 109, 106, 106, 107, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.752974574Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.752990082Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[1]R flush_numbers[0..12]: [175, 175, 176, 176, 181, 172, 181, 175, 176, 178, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.75300446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[1]R generation[0..12]: [106, 106, 106, 106, 109, 103, 109, 106, 106, 107, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753023149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753037987Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[2]R flush_numbers[0..12]: [175, 175, 176, 176, 181, 172, 181, 175, 176, 178, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753052175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[2]R generation[0..12]: [106, 106, 106, 106, 109, 103, 109, 106, 106, 107, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753066134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753086742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753100021Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753113199Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753126138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753138926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"":"downstairs","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753152235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753164344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"0","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753177562Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"1","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753190591Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"client":"2","":"downstairs","session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753203879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753218228Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"d61be264-b735-4126-98d0-81a8e2073260 is now active with session: e98e2dd6-db12-43ff-b02e-701619b87658","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:05:06.753231217Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753245545Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632,"session_id":"e98e2dd6-db12-43ff-b02e-701619b87658"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753258074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} {"msg":"Activated sub_volume d61be264-b735-4126-98d0-81a8e2073260","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:06.753269513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 50 len 4096 data: 6 02/30 Write block 28 len 28672 data: 7 9 10 9 10 11 12 03/30 Write block 132 len 20480 data: 5 6 5 9 9 04/30 Read block 251 len 36864 05/30 Write block 302 len 28672 data: 15 15 15 12 10 9 10 06/30 Write block 575 len 20480 data: 13 10 9 10 10 07/30 Write block 41 len 20480 data: 12 10 10 10 6 08/30 Write block 251 len 32768 data: 5 6 5 6 6 6 7 6 09/30 Read block 564 len 32768 10/30 Write block 347 len 28672 data: 7 8 6 8 8 10 12 11/30 Flush 12/30 Write block 60 len 4096 data: 12 13/30 Write block 130 len 32768 data: 6 5 6 7 6 10 10 8 14/30 Read block 121 len 28672 15/30 Write block 8 len 4096 data: 6 16/30 Read block 7 len 32768 17/30 Read block 510 len 32768 18/30 Write block 271 len 20480 data: 10 7 7 8 6 19/30 Flush 20/30 Flush 21/30 Write block 409 len 36864 data: 9 10 9 8 8 8 8 7 9 22/30 Flush 23/30 Write block 123 len 8192 data: 8 9 24/30 Read block 470 len 40960 25/30 Read block 261 len 36864 26/30 Write block 287 len 12288 data: 14 13 13 27/30 Write block 73 len 36864 data: 5 7 8 8 9 9 11 10 11 28/30 Read block 64 len 20480 29/30 Read block 38 len 12288 30/30 Read block 339 len 28672 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:06.773479575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 183 183 183 F F F 1 020-039 110 110 110 182 182 182 F F F 2 040-059 110 110 110 182 182 182 F F F 3 060-079 110 110 110 183 183 183 T T T 4 080-099 109 109 109 181 181 181 T T T 5 100-119 103 103 103 172 172 172 F F F 6 120-139 110 110 110 183 183 183 T T T 7 140-159 106 106 106 175 175 175 F F F 8 160-179 106 106 106 176 176 176 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 109 109 109 181 181 181 F F F 12 240-259 110 110 110 182 182 182 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 108 108 108 180 180 180 T T T 15 300-319 110 110 110 182 182 182 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 110 110 110 182 182 182 F F F 18 360-379 108 108 108 180 180 180 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 110 110 110 185 185 185 F F F 21 420-439 109 109 109 181 181 181 F F F 22 440-459 109 109 109 181 181 181 F F F 23 460-479 105 105 105 174 174 174 F F F 24 480-499 107 107 107 178 178 178 F F F 25 500-519 107 107 107 178 178 178 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 110 110 110 182 182 182 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 110, Max flush: 185 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 183 183 183 F F F 1 020-039 110 110 110 182 182 182 F F F 2 040-059 110 110 110 182 182 182 F F F 3 060-079 111 111 111 186 186 186 F F F 4 080-099 111 111 111 186 186 186 F F F 5 100-119 103 103 103 172 172 172 F F F 6 120-139 111 111 111 186 186 186 F F F 7 140-159 106 106 106 175 175 175 F F F 8 160-179 106 106 106 176 176 176 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 109 109 109 181 181 181 F F F 12 240-259 110 110 110 182 182 182 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 111 111 111 186 186 186 F F F 15 300-319 110 110 110 182 182 182 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 110 110 110 182 182 182 F F F 18 360-379 108 108 108 180 180 180 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 110 110 110 185 185 185 F F F 21 420-439 109 109 109 181 181 181 F F F 22 440-459 109 109 109 181 181 181 F F F 23 460-479 105 105 105 174 174 174 F F F 24 480-499 107 107 107 178 178 178 F F F 25 500-519 107 107 107 178 178 178 F F F 26 520-539 97 97 97 165 165 165 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 110 110 110 182 182 182 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 111, Max flush: 186 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.009416232Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.053582016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.054285853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.054328098Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.054344167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.054360115Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} {"msg":"Crucible e17bfbe5-b38f-4331-8ae2-ecce600fb0d8 has session id: cb7d47f8-683c-4f9d-a3b2-9187174c1740","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05507821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Upstairs opts: Upstairs UUID: e17bfbe5-b38f-4331-8ae2-ecce600fb0d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055127155Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Crucible stats registered with UUID: e17bfbe5-b38f-4331-8ae2-ecce600fb0d8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055151922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:11.05517138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055567649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055593776Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} {"msg":"e17bfbe5-b38f-4331-8ae2-ecce600fb0d8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05565642Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055679477Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"0","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055725952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"0","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055808924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"1","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05584971Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"1","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055875977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"2","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055905034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"2","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055930141Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.055955579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05604176Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"ds_connection connected from Ok(127.0.0.1:59313)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.056079006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"ds_connection connected from Ok(127.0.0.1:50273)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.056106523Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"ds_connection connected from Ok(127.0.0.1:35900)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05613382Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.056543197Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"0","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.056582573Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"1","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.057098649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"2","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.057247214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"1","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.0572859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"0","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.057699187Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"2","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.057872859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05795799Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[0]R flush_numbers[0..12]: [183, 182, 182, 186, 186, 172, 186, 175, 176, 178, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058201794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[0]R generation[0..12]: [110, 110, 110, 111, 111, 103, 111, 106, 106, 107, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058226342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05824213Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[1]R flush_numbers[0..12]: [183, 182, 182, 186, 186, 172, 186, 175, 176, 178, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058256569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[1]R generation[0..12]: [110, 110, 110, 111, 111, 103, 111, 106, 106, 107, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058270527Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058284546Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[2]R flush_numbers[0..12]: [183, 182, 182, 186, 186, 172, 186, 175, 176, 178, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058298324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[2]R generation[0..12]: [110, 110, 110, 111, 111, 103, 111, 106, 106, 107, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058312033Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058326011Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.05833904Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058352259Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058365047Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058382186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058403243Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058415162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"0","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058428441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"1","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058441499Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"client":"2","":"downstairs","session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058454468Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058466717Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"e17bfbe5-b38f-4331-8ae2-ecce600fb0d8 is now active with session: cb7d47f8-683c-4f9d-a3b2-9187174c1740","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058479405Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058491334Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640,"session_id":"cb7d47f8-683c-4f9d-a3b2-9187174c1740"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058503563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} {"msg":"Activated sub_volume e17bfbe5-b38f-4331-8ae2-ecce600fb0d8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:11.058514672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 20480 data: 9 9 8 8 7 02/30 Read block 116 len 32768 03/30 Flush 04/30 Write block 506 len 4096 data: 7 05/30 Write block 70 len 8192 data: 4 4 06/30 Write block 98 len 12288 data: 9 7 8 07/30 Read block 130 len 8192 08/30 Flush 09/30 Write block 184 len 16384 data: 9 8 9 12 10/30 Flush 11/30 Write block 500 len 28672 data: 8 9 10 9 8 6 8 12/30 Read block 577 len 40960 13/30 Flush 14/30 Write block 167 len 20480 data: 10 12 12 13 15 15/30 Read block 590 len 32768 16/30 Flush 17/30 Write block 348 len 24576 data: 9 7 9 9 11 13 18/30 Write block 126 len 28672 data: 8 9 8 7 7 6 7 19/30 Write block 477 len 4096 data: 10 20/30 Write block 535 len 12288 data: 6 8 10 21/30 Write block 102 len 8192 data: 9 6 22/30 Read block 130 len 12288 23/30 Read block 446 len 20480 24/30 Read block 513 len 4096 25/30 Read block 102 len 16384 26/30 Write block 297 len 28672 data: 12 12 13 11 13 16 16 27/30 Read block 23 len 4096 28/30 Read block 94 len 4096 29/30 Read block 127 len 32768 30/30 Write block 193 len 8192 data: 8 10 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:11.077109762Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 183 183 183 F F F 1 020-039 110 110 110 182 182 182 F F F 2 040-059 110 110 110 182 182 182 F F F 3 060-079 112 112 112 188 188 188 F F F 4 080-099 112 112 112 188 188 188 F F F 5 100-119 112 112 112 188 188 188 T T T 6 120-139 111 111 111 186 186 186 T T T 7 140-159 106 106 106 175 175 175 F F F 8 160-179 112 112 112 191 191 191 F F F 9 180-199 112 112 112 189 189 189 T T T 10 200-219 106 106 106 176 176 176 F F F 11 220-239 109 109 109 181 181 181 F F F 12 240-259 110 110 110 182 182 182 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 111 111 111 186 186 186 T T T 15 300-319 110 110 110 182 182 182 T T T 16 320-339 102 102 102 171 171 171 F F F 17 340-359 110 110 110 182 182 182 T T T 18 360-379 108 108 108 180 180 180 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 110 110 110 185 185 185 F F F 21 420-439 109 109 109 181 181 181 F F F 22 440-459 109 109 109 181 181 181 F F F 23 460-479 105 105 105 174 174 174 T T T 24 480-499 112 112 112 187 187 187 F F F 25 500-519 112 112 112 190 190 190 F F F 26 520-539 97 97 97 165 165 165 T T T 27 540-559 98 98 98 166 166 166 F F F 28 560-579 110 110 110 182 182 182 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 112, Max flush: 191 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 183 183 183 F F F 1 020-039 110 110 110 182 182 182 F F F 2 040-059 110 110 110 182 182 182 F F F 3 060-079 112 112 112 188 188 188 F F F 4 080-099 112 112 112 188 188 188 F F F 5 100-119 113 113 113 192 192 192 F F F 6 120-139 113 113 113 192 192 192 F F F 7 140-159 106 106 106 175 175 175 F F F 8 160-179 112 112 112 191 191 191 F F F 9 180-199 113 113 113 192 192 192 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 109 109 109 181 181 181 F F F 12 240-259 110 110 110 182 182 182 F F F 13 260-279 110 110 110 183 183 183 F F F 14 280-299 113 113 113 192 192 192 F F F 15 300-319 113 113 113 192 192 192 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 113 113 113 192 192 192 F F F 18 360-379 108 108 108 180 180 180 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 110 110 110 185 185 185 F F F 21 420-439 109 109 109 181 181 181 F F F 22 440-459 109 109 109 181 181 181 F F F 23 460-479 113 113 113 192 192 192 F F F 24 480-499 112 112 112 187 187 187 F F F 25 500-519 112 112 112 190 190 190 F F F 26 520-539 113 113 113 192 192 192 F F F 27 540-559 98 98 98 166 166 166 F F F 28 560-579 110 110 110 182 182 182 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 113, Max flush: 192 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.336154421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:15.379278224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.379654985Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.410608737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.411172958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.411207004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.411221853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.411236331Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} {"msg":"Crucible 1e13b74b-552f-4436-9f2c-411fe5b8c6a7 has session id: 875b5223-591e-4851-acf3-e7ad0332b42e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.411808392Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Upstairs opts: Upstairs UUID: 1e13b74b-552f-4436-9f2c-411fe5b8c6a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.411851457Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Crucible stats registered with UUID: 1e13b74b-552f-4436-9f2c-411fe5b8c6a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.411872215Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:15.411885574Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412444595Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412466923Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} {"msg":"1e13b74b-552f-4436-9f2c-411fe5b8c6a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412519167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412552314Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"0","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412605629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"0","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412636935Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"1","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412677531Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"1","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412706468Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"2","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412733745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"2","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412759653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.41278855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412816787Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44015)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412849883Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48194)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.412874541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64949)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.41307147Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.413382397Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"2","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.413535212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"1","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.413893314Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"0","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.414019931Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"2","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.414064316Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"1","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.414558075Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"0","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.414772583Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.4147942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[0]R flush_numbers[0..12]: [183, 182, 182, 188, 188, 192, 192, 175, 191, 192, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.41508628Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[0]R generation[0..12]: [110, 110, 110, 112, 112, 113, 113, 106, 112, 113, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415115787Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415132245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[1]R flush_numbers[0..12]: [183, 182, 182, 188, 188, 192, 192, 175, 191, 192, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415146864Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[1]R generation[0..12]: [110, 110, 110, 112, 112, 113, 113, 106, 112, 113, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415164392Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.41517892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[2]R flush_numbers[0..12]: [183, 182, 182, 188, 188, 192, 192, 175, 191, 192, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415192609Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[2]R generation[0..12]: [110, 110, 110, 112, 112, 113, 113, 106, 112, 113, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415206127Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415220136Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415233104Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415246123Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415267501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415280379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"":"downstairs","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415293508Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415305357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"0","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415318556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"1","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415331674Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"client":"2","":"downstairs","session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415344693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415358622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"1e13b74b-552f-4436-9f2c-411fe5b8c6a7 is now active with session: 875b5223-591e-4851-acf3-e7ad0332b42e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.41537147Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415383139Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648,"session_id":"875b5223-591e-4851-acf3-e7ad0332b42e"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415395268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} {"msg":"Activated sub_volume 1e13b74b-552f-4436-9f2c-411fe5b8c6a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:15.415406306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 8192 data: 9 6 02/30 Read block 294 len 40960 03/30 Flush 04/30 Write block 39 len 4096 data: 12 05/30 Read block 276 len 12288 06/30 Write block 378 len 28672 data: 8 9 12 15 15 17 15 07/30 Write block 258 len 12288 data: 7 9 8 08/30 Write block 497 len 4096 data: 10 09/30 Write block 94 len 12288 data: 10 9 10 10/30 Write block 551 len 4096 data: 6 11/30 Write block 265 len 20480 data: 9 9 9 11 11 12/30 Read block 37 len 12288 13/30 Read block 330 len 16384 14/30 Read block 450 len 20480 15/30 Read block 354 len 12288 16/30 Write block 365 len 12288 data: 8 8 8 17/30 Read block 447 len 24576 18/30 Read block 323 len 28672 19/30 Read block 568 len 20480 20/30 Read block 203 len 20480 21/30 Write block 290 len 12288 data: 17 17 15 22/30 Read block 256 len 36864 23/30 Read block 486 len 16384 24/30 Write block 448 len 32768 data: 7 9 9 10 12 12 10 10 25/30 Read block 176 len 40960 26/30 Write block 295 len 24576 data: 12 12 13 13 14 12 27/30 Write block 438 len 12288 data: 14 14 11 28/30 Write block 255 len 16384 data: 7 7 8 8 29/30 Write block 19 len 4096 data: 13 30/30 Read block 295 len 16384 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 1 Sent Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 0 5 Done 25 24 27 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:15.4335967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 183 183 183 T T T 1 020-039 110 110 110 182 182 182 T T T 2 040-059 114 114 114 193 193 193 F F F 3 060-079 112 112 112 188 188 188 F F F 4 080-099 112 112 112 188 188 188 T T T 5 100-119 113 113 113 192 192 192 F F F 6 120-139 113 113 113 192 192 192 F F F 7 140-159 106 106 106 175 175 175 F F F 8 160-179 112 112 112 191 191 191 F F F 9 180-199 113 113 113 192 192 192 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 109 109 109 181 181 181 F F F 12 240-259 110 110 110 182 182 182 T T T 13 260-279 110 110 110 183 183 183 T T T 14 280-299 113 113 113 192 192 192 T T T 15 300-319 113 113 113 192 192 192 T T T 16 320-339 102 102 102 171 171 171 F F F 17 340-359 113 113 113 192 192 192 F F F 18 360-379 108 108 108 180 180 180 T T T 19 380-399 109 109 109 181 181 181 T T T 20 400-419 110 110 110 185 185 185 F F F 21 420-439 109 109 109 181 181 181 T T T 22 440-459 109 109 109 181 181 181 T T T 23 460-479 113 113 113 192 192 192 F F F 24 480-499 112 112 112 187 187 187 T T T 25 500-519 112 112 112 190 190 190 F F F 26 520-539 113 113 113 192 192 192 F F F 27 540-559 98 98 98 166 166 166 T T T 28 560-579 110 110 110 182 182 182 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 114, Max flush: 193 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 194 194 194 F F F 1 020-039 115 115 115 194 194 194 F F F 2 040-059 114 114 114 193 193 193 F F F 3 060-079 112 112 112 188 188 188 F F F 4 080-099 115 115 115 194 194 194 F F F 5 100-119 113 113 113 192 192 192 F F F 6 120-139 113 113 113 192 192 192 F F F 7 140-159 106 106 106 175 175 175 F F F 8 160-179 112 112 112 191 191 191 F F F 9 180-199 113 113 113 192 192 192 F F F 10 200-219 106 106 106 176 176 176 F F F 11 220-239 109 109 109 181 181 181 F F F 12 240-259 115 115 115 194 194 194 F F F 13 260-279 115 115 115 194 194 194 F F F 14 280-299 115 115 115 194 194 194 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 102 102 102 171 171 171 F F F 17 340-359 113 113 113 192 192 192 F F F 18 360-379 115 115 115 194 194 194 F F F 19 380-399 115 115 115 194 194 194 F F F 20 400-419 110 110 110 185 185 185 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 113 113 113 192 192 192 F F F 24 480-499 115 115 115 194 194 194 F F F 25 500-519 112 112 112 190 190 190 F F F 26 520-539 113 113 113 192 192 192 F F F 27 540-559 115 115 115 194 194 194 F F F 28 560-579 110 110 110 182 182 182 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 115, Max flush: 194 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.743261668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:19.787188847Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.787577566Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.818627278Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.819199229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.819241794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.819259622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.81927749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} {"msg":"Crucible 90f21e06-7267-4ef3-8c71-2bc280f808db has session id: 0c047003-f3c1-4c11-95eb-2a390a5ab443","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.81985492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Upstairs opts: Upstairs UUID: 90f21e06-7267-4ef3-8c71-2bc280f808db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.819900115Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Crucible stats registered with UUID: 90f21e06-7267-4ef3-8c71-2bc280f808db","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.819919813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:19.819935712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820369067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820392874Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} {"msg":"90f21e06-7267-4ef3-8c71-2bc280f808db active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820458967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820477065Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"0","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820504972Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"0","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.82052769Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"1","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820555447Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"1","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820583604Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"2","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820613201Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"2","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820641958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820672535Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820703312Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"ds_connection connected from Ok(127.0.0.1:47979)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820732989Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"ds_connection connected from Ok(127.0.0.1:41234)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820759466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"ds_connection connected from Ok(127.0.0.1:39054)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.820783883Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.821458703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"2","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.82149131Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"0","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.821507978Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"1","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.822166729Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"0","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.822303995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"1","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.822334251Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"2","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.822764257Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.822938708Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[0]R flush_numbers[0..12]: [194, 194, 193, 188, 194, 192, 192, 175, 191, 192, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823014131Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[0]R generation[0..12]: [115, 115, 114, 112, 115, 113, 113, 106, 112, 113, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823035258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823050227Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[1]R flush_numbers[0..12]: [194, 194, 193, 188, 194, 192, 192, 175, 191, 192, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823064875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[1]R generation[0..12]: [115, 115, 114, 112, 115, 113, 113, 106, 112, 113, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823078864Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823092703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[2]R flush_numbers[0..12]: [194, 194, 193, 188, 194, 192, 192, 175, 191, 192, 176, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823110181Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[2]R generation[0..12]: [115, 115, 114, 112, 115, 113, 113, 106, 112, 113, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823124829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823138758Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823151696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823164755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823178283Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:19.823192022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"":"downstairs","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.82320872Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823220819Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"0","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823234268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"1","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823247236Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"client":"2","":"downstairs","session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823260335Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823272384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"90f21e06-7267-4ef3-8c71-2bc280f808db is now active with session: 0c047003-f3c1-4c11-95eb-2a390a5ab443","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823284912Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823296881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656,"session_id":"0c047003-f3c1-4c11-95eb-2a390a5ab443"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.82330888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} {"msg":"Activated sub_volume 90f21e06-7267-4ef3-8c71-2bc280f808db","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:19.823322379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 59 len 12288 data: 14 13 11 02/30 Write block 557 len 16384 data: 10 9 9 9 03/30 Read block 30 len 32768 04/30 Write block 539 len 40960 data: 8 9 9 8 6 6 9 11 10 8 05/30 Write block 106 len 28672 data: 7 8 7 7 7 9 9 06/30 Write block 286 len 16384 data: 13 15 14 14 07/30 Read block 351 len 12288 08/30 Read block 206 len 16384 09/30 Write block 544 len 8192 data: 7 10 10/30 Read block 304 len 28672 11/30 Read block 580 len 20480 12/30 Write block 529 len 20480 data: 7 6 5 4 3 13/30 Read block 371 len 20480 14/30 Read block 149 len 20480 15/30 Read block 58 len 8192 16/30 Read block 292 len 20480 17/30 Write block 217 len 32768 data: 11 12 11 8 6 9 6 6 18/30 Write block 328 len 28672 data: 8 9 7 8 7 5 5 19/30 Read block 489 len 24576 20/30 Write block 152 len 8192 data: 8 9 21/30 Write block 411 len 32768 data: 10 9 9 9 9 8 10 11 22/30 Flush 23/30 Flush 24/30 Write block 50 len 4096 data: 7 25/30 Write block 466 len 12288 data: 4 5 6 26/30 Write block 354 len 28672 data: 10 12 11 13 12 8 8 27/30 Read block 59 len 24576 28/30 Read block 260 len 8192 29/30 Write block 320 len 24576 data: 9 10 11 9 9 8 30/30 Read block 424 len 32768 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 6 7 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:19.843662287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 194 194 194 F F F 1 020-039 115 115 115 194 194 194 F F F 2 040-059 116 116 116 195 195 195 T T T 3 060-079 116 116 116 195 195 195 F F F 4 080-099 115 115 115 194 194 194 F F F 5 100-119 116 116 116 195 195 195 F F F 6 120-139 113 113 113 192 192 192 F F F 7 140-159 116 116 116 195 195 195 F F F 8 160-179 112 112 112 191 191 191 F F F 9 180-199 113 113 113 192 192 192 F F F 10 200-219 116 116 116 195 195 195 F F F 11 220-239 116 116 116 195 195 195 F F F 12 240-259 115 115 115 194 194 194 F F F 13 260-279 115 115 115 194 194 194 F F F 14 280-299 116 116 116 195 195 195 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 116 116 116 195 195 195 T T T 17 340-359 113 113 113 192 192 192 T T T 18 360-379 115 115 115 194 194 194 T T T 19 380-399 115 115 115 194 194 194 F F F 20 400-419 116 116 116 195 195 195 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 113 113 113 192 192 192 T T T 24 480-499 115 115 115 194 194 194 F F F 25 500-519 112 112 112 190 190 190 F F F 26 520-539 116 116 116 195 195 195 F F F 27 540-559 116 116 116 195 195 195 F F F 28 560-579 116 116 116 195 195 195 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 116, Max flush: 195 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 194 194 194 F F F 1 020-039 115 115 115 194 194 194 F F F 2 040-059 117 117 117 196 196 196 F F F 3 060-079 116 116 116 195 195 195 F F F 4 080-099 115 115 115 194 194 194 F F F 5 100-119 116 116 116 195 195 195 F F F 6 120-139 113 113 113 192 192 192 F F F 7 140-159 116 116 116 195 195 195 F F F 8 160-179 112 112 112 191 191 191 F F F 9 180-199 113 113 113 192 192 192 F F F 10 200-219 116 116 116 195 195 195 F F F 11 220-239 116 116 116 195 195 195 F F F 12 240-259 115 115 115 194 194 194 F F F 13 260-279 115 115 115 194 194 194 F F F 14 280-299 116 116 116 195 195 195 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 117 117 117 196 196 196 F F F 17 340-359 117 117 117 196 196 196 F F F 18 360-379 117 117 117 196 196 196 F F F 19 380-399 115 115 115 194 194 194 F F F 20 400-419 116 116 116 195 195 195 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 117 117 117 196 196 196 F F F 24 480-499 115 115 115 194 194 194 F F F 25 500-519 112 112 112 190 190 190 F F F 26 520-539 116 116 116 195 195 195 F F F 27 540-559 116 116 116 195 195 195 F F F 28 560-579 116 116 116 195 195 195 F F F 29 580-599 104 104 104 173 173 173 F F F Max gen: 117, Max flush: 196 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.110658047Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:24.154272728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.154666387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.185503592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.186100179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.186134966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.186153104Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.186167732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} {"msg":"Crucible 25ee4b10-a384-4d35-98f3-62cc1ed4e25c has session id: 198d70f3-a11c-4a0b-b627-d695e1b647a2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.186778328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Upstairs opts: Upstairs UUID: 25ee4b10-a384-4d35-98f3-62cc1ed4e25c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.186825114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Crucible stats registered with UUID: 25ee4b10-a384-4d35-98f3-62cc1ed4e25c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.186852071Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:24.186868489Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187285796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187311363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} {"msg":"25ee4b10-a384-4d35-98f3-62cc1ed4e25c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187374026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187392144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"0","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187420652Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"0","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.18743548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"1","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187450909Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"1","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187474726Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"2","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187560997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"2","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187596123Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187623751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187647308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61541)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187672445Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34690)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187773755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:35492)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.187809191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.188421517Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"2","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.188453224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"1","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.188554433Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"0","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189096087Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"1","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189125194Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"2","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189190377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"0","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189727981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189759258Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[0]R flush_numbers[0..12]: [194, 194, 196, 195, 194, 195, 192, 195, 191, 192, 195, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189780446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[0]R generation[0..12]: [115, 115, 117, 116, 115, 116, 113, 116, 112, 113, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189796134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189810692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[1]R flush_numbers[0..12]: [194, 194, 196, 195, 194, 195, 192, 195, 191, 192, 195, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189824431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[1]R generation[0..12]: [115, 115, 117, 116, 115, 116, 113, 116, 112, 113, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189838249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189852268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[2]R flush_numbers[0..12]: [194, 194, 196, 195, 194, 195, 192, 195, 191, 192, 195, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189866086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[2]R generation[0..12]: [115, 115, 117, 116, 115, 116, 113, 116, 112, 113, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189879495Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189893214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189906093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189919181Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.18993197Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189944928Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"":"downstairs","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189960317Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189972666Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"0","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189985734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"1","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.189998683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"client":"2","":"downstairs","session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.190019561Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:24.190043388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"25ee4b10-a384-4d35-98f3-62cc1ed4e25c is now active with session: 198d70f3-a11c-4a0b-b627-d695e1b647a2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.190062306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.190074605Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664,"session_id":"198d70f3-a11c-4a0b-b627-d695e1b647a2"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.190086723Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} {"msg":"Activated sub_volume 25ee4b10-a384-4d35-98f3-62cc1ed4e25c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:24.190100212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 354 len 8192 data: 11 13 02/30 Read block 249 len 36864 03/30 Read block 256 len 36864 04/30 Read block 397 len 28672 05/30 Write block 293 len 24576 data: 12 13 13 13 14 14 06/30 Write block 144 len 28672 data: 7 7 9 10 10 9 7 07/30 Flush 08/30 Write block 169 len 40960 data: 13 14 16 10 12 10 12 16 16 12 09/30 Read block 581 len 28672 10/30 Read block 127 len 40960 11/30 Write block 477 len 16384 data: 11 10 11 9 12/30 Write block 118 len 12288 data: 10 13 11 13/30 Read block 488 len 20480 14/30 Write block 490 len 16384 data: 10 8 9 9 15/30 Write block 402 len 12288 data: 10 13 12 16/30 Read block 544 len 4096 17/30 Write block 337 len 12288 data: 7 8 8 18/30 Write block 242 len 24576 data: 14 13 11 8 7 7 19/30 Flush 20/30 Write block 187 len 8192 data: 13 12 21/30 Flush 22/30 Write block 502 len 16384 data: 11 10 9 7 23/30 Write block 499 len 16384 data: 9 9 10 12 24/30 Read block 448 len 8192 25/30 Write block 84 len 28672 data: 12 14 11 12 12 10 10 26/30 Write block 367 len 36864 data: 9 6 7 9 7 8 6 5 7 27/30 Write block 587 len 12288 data: 9 7 9 28/30 Read block 60 len 16384 29/30 Write block 375 len 12288 data: 8 5 7 30/30 Read block 504 len 24576 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 8 9 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:24.209890168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 194 194 194 F F F 1 020-039 115 115 115 194 194 194 F F F 2 040-059 117 117 117 196 196 196 F F F 3 060-079 116 116 116 195 195 195 F F F 4 080-099 115 115 115 194 194 194 T T T 5 100-119 118 118 118 198 198 198 F F F 6 120-139 118 118 118 198 198 198 F F F 7 140-159 118 118 118 197 197 197 F F F 8 160-179 118 118 118 198 198 198 F F F 9 180-199 118 118 118 199 199 199 F F F 10 200-219 116 116 116 195 195 195 F F F 11 220-239 116 116 116 195 195 195 F F F 12 240-259 118 118 118 198 198 198 F F F 13 260-279 115 115 115 194 194 194 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 118 118 118 198 198 198 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 117 117 117 196 196 196 T T T 19 380-399 115 115 115 194 194 194 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 118 118 118 198 198 198 T T T 25 500-519 112 112 112 190 190 190 T T T 26 520-539 116 116 116 195 195 195 F F F 27 540-559 116 116 116 195 195 195 F F F 28 560-579 116 116 116 195 195 195 F F F 29 580-599 104 104 104 173 173 173 T T T Max gen: 118, Max flush: 199 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 194 194 194 F F F 1 020-039 115 115 115 194 194 194 F F F 2 040-059 117 117 117 196 196 196 F F F 3 060-079 116 116 116 195 195 195 F F F 4 080-099 119 119 119 200 200 200 F F F 5 100-119 118 118 118 198 198 198 F F F 6 120-139 118 118 118 198 198 198 F F F 7 140-159 118 118 118 197 197 197 F F F 8 160-179 118 118 118 198 198 198 F F F 9 180-199 118 118 118 199 199 199 F F F 10 200-219 116 116 116 195 195 195 F F F 11 220-239 116 116 116 195 195 195 F F F 12 240-259 118 118 118 198 198 198 F F F 13 260-279 115 115 115 194 194 194 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 118 118 118 198 198 198 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 119 119 119 200 200 200 F F F 19 380-399 115 115 115 194 194 194 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 119 119 119 200 200 200 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 116 116 116 195 195 195 F F F 27 540-559 116 116 116 195 195 195 F F F 28 560-579 116 116 116 195 195 195 F F F 29 580-599 119 119 119 200 200 200 F F F Max gen: 119, Max flush: 200 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.47542237Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:28.519715381Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.520132277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.551211586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.551788966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.551823582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.551838161Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.551852439Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} {"msg":"Crucible 78a2cf64-b01f-4f28-9c2d-bd4e18d640c3 has session id: 349b4f63-3eaa-44e6-a875-227af93f2d58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.552432519Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Upstairs opts: Upstairs UUID: 78a2cf64-b01f-4f28-9c2d-bd4e18d640c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.552468425Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Crucible stats registered with UUID: 78a2cf64-b01f-4f28-9c2d-bd4e18d640c3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.552484513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:28.552501282Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.552967953Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.552991951Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} {"msg":"78a2cf64-b01f-4f28-9c2d-bd4e18d640c3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553041305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553058673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"0","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553086641Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"0","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553100669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"1","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553114828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"1","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553136085Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"2","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553161383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"2","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553176161Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553193099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553218617Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:54224)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553251384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:54749)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.55327945Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:57605)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.553410537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.554115923Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"1","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.554158069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"2","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.554261498Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"0","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.554779004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"2","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.55481909Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"1","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.554873134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"0","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555379891Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.5554888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[0]R flush_numbers[0..12]: [194, 194, 196, 195, 200, 198, 198, 197, 198, 199, 195, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555555633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[0]R generation[0..12]: [115, 115, 117, 116, 119, 118, 118, 118, 118, 118, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.55558447Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555600168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[1]R flush_numbers[0..12]: [194, 194, 196, 195, 200, 198, 198, 197, 198, 199, 195, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555614537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[1]R generation[0..12]: [115, 115, 117, 116, 119, 118, 118, 118, 118, 118, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555628346Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555649713Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[2]R flush_numbers[0..12]: [194, 194, 196, 195, 200, 198, 198, 197, 198, 199, 195, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555671241Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[2]R generation[0..12]: [115, 115, 117, 116, 119, 118, 118, 118, 118, 118, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555689559Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555714037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555738924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555752993Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.55577431Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555805647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"":"downstairs","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555831715Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555846893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"0","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555860112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"1","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.55587728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"client":"2","":"downstairs","session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555900497Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555925095Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"78a2cf64-b01f-4f28-9c2d-bd4e18d640c3 is now active with session: 349b4f63-3eaa-44e6-a875-227af93f2d58","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555944403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555959621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672,"session_id":"349b4f63-3eaa-44e6-a875-227af93f2d58"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.555980569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} {"msg":"Activated sub_volume 78a2cf64-b01f-4f28-9c2d-bd4e18d640c3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:28.556001897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 135 len 20480 data: 11 11 9 9 9 02/30 Write block 561 len 12288 data: 9 7 8 03/30 Read block 207 len 40960 04/30 Write block 17 len 4096 data: 13 05/30 Write block 142 len 36864 data: 7 7 8 8 10 11 11 10 8 06/30 Write block 253 len 40960 data: 6 7 8 8 9 9 10 9 8 5 07/30 Read block 55 len 28672 08/30 Write block 4 len 24576 data: 7 7 9 9 7 8 09/30 Write block 6 len 8192 data: 10 10 10/30 Read block 412 len 36864 11/30 Write block 0 len 40960 data: 4 6 7 7 8 8 11 11 8 9 12/30 Write block 192 len 28672 data: 10 9 11 10 11 9 8 13/30 Read block 404 len 28672 14/30 Read block 163 len 4096 15/30 Read block 499 len 12288 16/30 Read block 490 len 4096 17/30 Read block 406 len 28672 18/30 Read block 37 len 20480 19/30 Write block 133 len 12288 data: 8 7 12 20/30 Write block 521 len 12288 data: 6 8 9 21/30 Write block 45 len 8192 data: 7 6 22/30 Flush 23/30 Write block 498 len 8192 data: 9 10 24/30 Write block 369 len 8192 data: 8 10 25/30 Flush 26/30 Write block 320 len 24576 data: 10 11 12 10 10 9 27/30 Write block 208 len 24576 data: 9 9 8 11 9 13 28/30 Read block 35 len 16384 29/30 Read block 382 len 28672 30/30 Read block 184 len 12288 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:28.575864055Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 201 201 201 F F F 1 020-039 115 115 115 194 194 194 F F F 2 040-059 120 120 120 201 201 201 F F F 3 060-079 116 116 116 195 195 195 F F F 4 080-099 119 119 119 200 200 200 F F F 5 100-119 118 118 118 198 198 198 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 118 118 118 198 198 198 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 116 116 116 195 195 195 T T T 11 220-239 116 116 116 195 195 195 F F F 12 240-259 120 120 120 201 201 201 F F F 13 260-279 120 120 120 201 201 201 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 118 118 118 198 198 198 T T T 17 340-359 118 118 118 197 197 197 F F F 18 360-379 120 120 120 202 202 202 F F F 19 380-399 115 115 115 194 194 194 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 120 120 120 202 202 202 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 120 120 120 201 201 201 F F F 27 540-559 116 116 116 195 195 195 F F F 28 560-579 120 120 120 201 201 201 F F F 29 580-599 119 119 119 200 200 200 F F F Max gen: 120, Max flush: 202 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 201 201 201 F F F 1 020-039 115 115 115 194 194 194 F F F 2 040-059 120 120 120 201 201 201 F F F 3 060-079 116 116 116 195 195 195 F F F 4 080-099 119 119 119 200 200 200 F F F 5 100-119 118 118 118 198 198 198 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 118 118 118 198 198 198 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 121 121 121 203 203 203 F F F 11 220-239 116 116 116 195 195 195 F F F 12 240-259 120 120 120 201 201 201 F F F 13 260-279 120 120 120 201 201 201 F F F 14 280-299 118 118 118 197 197 197 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 121 121 121 203 203 203 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 120 120 120 202 202 202 F F F 19 380-399 115 115 115 194 194 194 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 120 120 120 202 202 202 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 120 120 120 201 201 201 F F F 27 540-559 116 116 116 195 195 195 F F F 28 560-579 120 120 120 201 201 201 F F F 29 580-599 119 119 119 200 200 200 F F F Max gen: 121, Max flush: 203 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.826724587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.872220162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.872960455Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.873025318Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.873047936Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.873063614Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} {"msg":"Crucible 19e3b970-f0cb-47f3-bbe0-370fdbc3d8a8 has session id: 7497e220-61af-4bd1-b597-54067df458ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.873655062Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Upstairs opts: Upstairs UUID: 19e3b970-f0cb-47f3-bbe0-370fdbc3d8a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.873691609Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Crucible stats registered with UUID: 19e3b970-f0cb-47f3-bbe0-370fdbc3d8a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.873707497Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:32.873720216Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874133223Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874166329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} {"msg":"19e3b970-f0cb-47f3-bbe0-370fdbc3d8a8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874224133Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.87425113Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"0","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874281107Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"0","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874298765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"1","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874325772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"1","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.87435134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"2","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874376137Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"2","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874408824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874437101Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874465398Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:53950)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874494995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:50436)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874522062Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:53138)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.874548439Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.875329738Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"0","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.875370734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"2","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.87540348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"1","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.875994929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"1","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876036194Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"2","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876068691Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"0","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.87664629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876673358Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[0]R flush_numbers[0..12]: [201, 194, 201, 195, 200, 198, 201, 201, 198, 201, 203, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876705824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[0]R generation[0..12]: [120, 115, 120, 116, 119, 118, 120, 120, 118, 120, 121, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876722853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876737601Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[1]R flush_numbers[0..12]: [201, 194, 201, 195, 200, 198, 201, 201, 198, 201, 203, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.87675169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[1]R generation[0..12]: [120, 115, 120, 116, 119, 118, 120, 120, 118, 120, 121, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876765658Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876779567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[2]R flush_numbers[0..12]: [201, 194, 201, 195, 200, 198, 201, 201, 198, 201, 203, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876793125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[2]R generation[0..12]: [120, 115, 120, 116, 119, 118, 120, 120, 118, 120, 121, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876806514Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876820252Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876833141Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.87684621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876859008Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876871807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"":"downstairs","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876885246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876899074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"0","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:32.876912563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"1","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876930741Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"client":"2","":"downstairs","session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876944099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876956348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"19e3b970-f0cb-47f3-bbe0-370fdbc3d8a8 is now active with session: 7497e220-61af-4bd1-b597-54067df458ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876969057Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876980806Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680,"session_id":"7497e220-61af-4bd1-b597-54067df458ef"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.876992785Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} {"msg":"Activated sub_volume 19e3b970-f0cb-47f3-bbe0-370fdbc3d8a8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:32.877003833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 82 len 36864 data: 11 14 13 15 12 13 13 11 11 02/30 Read block 283 len 4096 03/30 Read block 356 len 36864 04/30 Read block 326 len 20480 05/30 Write block 520 len 4096 data: 7 06/30 Write block 277 len 40960 data: 9 7 8 11 11 11 13 14 12 14 07/30 Write block 523 len 16384 data: 10 10 8 8 08/30 Write block 287 len 12288 data: 16 15 15 09/30 Read block 76 len 8192 10/30 Read block 203 len 28672 11/30 Read block 377 len 32768 12/30 Read block 202 len 32768 13/30 Read block 161 len 24576 14/30 Read block 510 len 24576 15/30 Read block 327 len 20480 16/30 Write block 133 len 20480 data: 9 8 13 12 10 17/30 Write block 38 len 24576 data: 12 13 12 13 11 11 18/30 Write block 57 len 20480 data: 10 13 15 14 12 19/30 Flush 20/30 Flush 21/30 Write block 143 len 12288 data: 8 9 9 22/30 Read block 516 len 36864 23/30 Read block 88 len 8192 24/30 Read block 164 len 16384 25/30 Write block 2 len 4096 data: 8 26/30 Flush 27/30 Write block 559 len 32768 data: 10 10 10 8 9 9 9 10 28/30 Write block 584 len 28672 data: 7 9 8 10 8 10 8 29/30 Write block 204 len 4096 data: 9 30/30 Write block 54 len 24576 data: 8 11 9 11 14 16 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:32.896294332Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 206 206 206 F F F 1 020-039 122 122 122 204 204 204 F F F 2 040-059 122 122 122 204 204 204 T T T 3 060-079 122 122 122 204 204 204 F F F 4 080-099 122 122 122 204 204 204 F F F 5 100-119 118 118 118 198 198 198 F F F 6 120-139 122 122 122 204 204 204 F F F 7 140-159 122 122 122 206 206 206 F F F 8 160-179 118 118 118 198 198 198 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 121 121 121 203 203 203 T T T 11 220-239 116 116 116 195 195 195 F F F 12 240-259 120 120 120 201 201 201 F F F 13 260-279 122 122 122 204 204 204 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 121 121 121 203 203 203 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 120 120 120 202 202 202 F F F 19 380-399 115 115 115 194 194 194 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 120 120 120 202 202 202 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 122 122 122 204 204 204 F F F 27 540-559 116 116 116 195 195 195 T T T 28 560-579 120 120 120 201 201 201 T T T 29 580-599 119 119 119 200 200 200 T T T Max gen: 122, Max flush: 206 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 206 206 206 F F F 1 020-039 122 122 122 204 204 204 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 122 122 122 204 204 204 F F F 4 080-099 122 122 122 204 204 204 F F F 5 100-119 118 118 118 198 198 198 F F F 6 120-139 122 122 122 204 204 204 F F F 7 140-159 122 122 122 206 206 206 F F F 8 160-179 118 118 118 198 198 198 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 123 123 123 207 207 207 F F F 11 220-239 116 116 116 195 195 195 F F F 12 240-259 120 120 120 201 201 201 F F F 13 260-279 122 122 122 204 204 204 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 115 115 115 194 194 194 F F F 16 320-339 121 121 121 203 203 203 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 120 120 120 202 202 202 F F F 19 380-399 115 115 115 194 194 194 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 F F F 23 460-479 118 118 118 198 198 198 F F F 24 480-499 120 120 120 202 202 202 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 122 122 122 204 204 204 F F F 27 540-559 123 123 123 207 207 207 F F F 28 560-579 123 123 123 207 207 207 F F F 29 580-599 123 123 123 207 207 207 F F F Max gen: 123, Max flush: 207 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.134613301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:37.178023974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.178418382Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.210160732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.210774478Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.210818054Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.210835052Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.21085307Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} {"msg":"Crucible 743744d4-8bce-40e9-93fe-5ab533841291 has session id: d42166c1-8572-4d39-bca0-2145056fef52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.21143347Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Upstairs opts: Upstairs UUID: 743744d4-8bce-40e9-93fe-5ab533841291, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.211473365Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Crucible stats registered with UUID: 743744d4-8bce-40e9-93fe-5ab533841291","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.211492723Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:37.211508222Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.211953495Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.211982922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} {"msg":"743744d4-8bce-40e9-93fe-5ab533841291 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212027717Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212042856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"0","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212067344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"0","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212080672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"1","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212094441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"1","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212107189Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"2","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212121038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"2","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212134226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212148095Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212170973Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"ds_connection connected from Ok(127.0.0.1:62160)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212222807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"ds_connection connected from Ok(127.0.0.1:48381)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212393539Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"ds_connection connected from Ok(127.0.0.1:58679)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212432825Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.212979458Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"2","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.213014155Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"1","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.213079468Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"0","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.213595974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"1","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.213669426Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"2","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.213699003Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"0","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214229488Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214265794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[0]R flush_numbers[0..12]: [206, 204, 207, 204, 204, 198, 204, 206, 198, 201, 207, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214290781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[0]R generation[0..12]: [122, 122, 123, 122, 122, 118, 122, 122, 118, 120, 123, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.21430801Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214322968Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[1]R flush_numbers[0..12]: [206, 204, 207, 204, 204, 198, 204, 206, 198, 201, 207, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214337077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[1]R generation[0..12]: [122, 122, 123, 122, 122, 118, 122, 122, 118, 120, 123, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214350705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214364544Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[2]R flush_numbers[0..12]: [206, 204, 207, 204, 204, 198, 204, 206, 198, 201, 207, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214378132Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[2]R generation[0..12]: [122, 122, 123, 122, 122, 118, 122, 122, 118, 120, 123, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214395251Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214410619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214434077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214508919Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214529997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214553644Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"":"downstairs","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214571772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214584181Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"0","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214597549Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"1","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214610698Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"client":"2","":"downstairs","session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214628966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214651344Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"743744d4-8bce-40e9-93fe-5ab533841291 is now active with session: d42166c1-8572-4d39-bca0-2145056fef52","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214665853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214677971Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688,"session_id":"d42166c1-8572-4d39-bca0-2145056fef52"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.21469002Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} {"msg":"Activated sub_volume 743744d4-8bce-40e9-93fe-5ab533841291","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:37.214708438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 492 len 28672 data: 10 10 9 8 10 11 10 02/30 Flush 03/30 Flush 04/30 Write block 11 len 12288 data: 11 12 11 05/30 Write block 376 len 36864 data: 6 8 9 10 13 16 16 18 16 06/30 Read block 299 len 36864 07/30 Read block 138 len 12288 08/30 Write block 139 len 12288 data: 10 9 9 09/30 Flush 10/30 Write block 582 len 8192 data: 5 5 11/30 Write block 26 len 40960 data: 7 6 8 10 11 10 11 12 13 11 12/30 Write block 464 len 16384 data: 8 6 5 6 13/30 Read block 423 len 24576 14/30 Read block 28 len 8192 15/30 Read block 588 len 12288 16/30 Write block 181 len 28672 data: 8 9 8 10 9 10 14 17/30 Read block 73 len 32768 18/30 Read block 263 len 32768 19/30 Read block 402 len 12288 20/30 Read block 24 len 24576 21/30 Read block 241 len 40960 22/30 Write block 175 len 28672 data: 13 17 17 13 10 7 9 23/30 Write block 212 len 20480 data: 10 14 13 12 10 24/30 Write block 35 len 20480 data: 12 11 12 13 14 25/30 Write block 318 len 24576 data: 10 7 11 12 13 11 26/30 Read block 322 len 12288 27/30 Read block 280 len 8192 28/30 Read block 461 len 40960 29/30 Read block 329 len 24576 30/30 Write block 444 len 16384 data: 8 11 10 10 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 2 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:37.235477862Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 122 122 122 204 204 204 T T T 2 040-059 123 123 123 207 207 207 F F F 3 060-079 122 122 122 204 204 204 F F F 4 080-099 122 122 122 204 204 204 F F F 5 100-119 118 118 118 198 198 198 F F F 6 120-139 124 124 124 210 210 210 F F F 7 140-159 124 124 124 210 210 210 F F F 8 160-179 118 118 118 198 198 198 T T T 9 180-199 120 120 120 201 201 201 T T T 10 200-219 123 123 123 207 207 207 T T T 11 220-239 116 116 116 195 195 195 F F F 12 240-259 120 120 120 201 201 201 F F F 13 260-279 122 122 122 204 204 204 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 115 115 115 194 194 194 T T T 16 320-339 121 121 121 203 203 203 T T T 17 340-359 118 118 118 197 197 197 F F F 18 360-379 124 124 124 210 210 210 F F F 19 380-399 124 124 124 210 210 210 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 115 115 115 194 194 194 T T T 23 460-479 118 118 118 198 198 198 T T T 24 480-499 124 124 124 208 208 208 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 122 122 122 204 204 204 F F F 27 540-559 123 123 123 207 207 207 F F F 28 560-579 123 123 123 207 207 207 F F F 29 580-599 123 123 123 207 207 207 T T T Max gen: 124, Max flush: 210 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 122 122 122 204 204 204 F F F 4 080-099 122 122 122 204 204 204 F F F 5 100-119 118 118 118 198 198 198 F F F 6 120-139 124 124 124 210 210 210 F F F 7 140-159 124 124 124 210 210 210 F F F 8 160-179 125 125 125 211 211 211 F F F 9 180-199 125 125 125 211 211 211 F F F 10 200-219 125 125 125 211 211 211 F F F 11 220-239 116 116 116 195 195 195 F F F 12 240-259 120 120 120 201 201 201 F F F 13 260-279 122 122 122 204 204 204 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 125 125 125 211 211 211 F F F 16 320-339 125 125 125 211 211 211 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 124 124 124 210 210 210 F F F 19 380-399 124 124 124 210 210 210 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 115 115 115 194 194 194 F F F 22 440-459 125 125 125 211 211 211 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 122 122 122 204 204 204 F F F 27 540-559 123 123 123 207 207 207 F F F 28 560-579 123 123 123 207 207 207 F F F 29 580-599 125 125 125 211 211 211 F F F Max gen: 125, Max flush: 211 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.525972651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:41.572243175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.572643444Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.604587502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.605350803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.605387929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.605402037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.605416516Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} {"msg":"Crucible 48be2d0b-ec2f-4bb9-b0a9-5ba2b56e9e00 has session id: 3c2016e3-8844-4b5c-b838-55a5b8e16170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606008024Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Upstairs opts: Upstairs UUID: 48be2d0b-ec2f-4bb9-b0a9-5ba2b56e9e00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.60605026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Crucible stats registered with UUID: 48be2d0b-ec2f-4bb9-b0a9-5ba2b56e9e00","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606067578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:41.606080287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606581925Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606605362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} {"msg":"48be2d0b-ec2f-4bb9-b0a9-5ba2b56e9e00 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606690323Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606711531Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"0","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606740198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"0","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606761216Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"1","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606794123Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"1","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.60681989Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"2","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606848697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"2","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606875434Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606901201Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606928558Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"ds_connection connected from Ok(127.0.0.1:58198)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.606957435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"ds_connection connected from Ok(127.0.0.1:40710)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.607002791Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"ds_connection connected from Ok(127.0.0.1:59988)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.607037457Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.607896937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"1","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.607939083Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"2","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.608112655Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"0","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.608610823Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"1","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.60863978Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"2","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.608726191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"0","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609263815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609289822Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[0]R flush_numbers[0..12]: [210, 211, 207, 204, 204, 198, 210, 210, 211, 211, 211, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609353685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[0]R generation[0..12]: [124, 125, 123, 122, 122, 118, 124, 124, 125, 125, 125, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609374433Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609389492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[1]R flush_numbers[0..12]: [210, 211, 207, 204, 204, 198, 210, 210, 211, 211, 211, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.60940325Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[1]R generation[0..12]: [124, 125, 123, 122, 122, 118, 124, 124, 125, 125, 125, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609417079Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609431117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[2]R flush_numbers[0..12]: [210, 211, 207, 204, 204, 198, 210, 210, 211, 211, 211, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609444906Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[2]R generation[0..12]: [124, 125, 123, 122, 122, 118, 124, 124, 125, 125, 125, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609458454Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609472183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609484922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.60949795Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609515029Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-03-18T05:05:41.609528507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"":"downstairs","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609548305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609560364Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"0","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609573313Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"1","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609586071Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"client":"2","":"downstairs","session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.60959909Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609611169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"48be2d0b-ec2f-4bb9-b0a9-5ba2b56e9e00 is now active with session: 3c2016e3-8844-4b5c-b838-55a5b8e16170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609623747Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609635556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696,"session_id":"3c2016e3-8844-4b5c-b838-55a5b8e16170"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609647725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} {"msg":"Activated sub_volume 48be2d0b-ec2f-4bb9-b0a9-5ba2b56e9e00","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:41.609658734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 111 len 40960 data: 10 10 9 8 10 9 9 11 14 12 02/30 Read block 335 len 16384 03/30 Write block 270 len 28672 data: 11 11 8 8 9 7 8 04/30 Flush 05/30 Write block 530 len 40960 data: 7 6 5 4 4 7 9 11 10 9 06/30 Read block 339 len 28672 07/30 Write block 158 len 12288 data: 12 12 13 08/30 Write block 425 len 32768 data: 10 10 12 11 11 10 11 12 09/30 Flush 10/30 Write block 558 len 32768 data: 10 11 11 11 9 10 10 10 11/30 Write block 88 len 32768 data: 14 12 12 11 13 10 11 10 12/30 Write block 446 len 28672 data: 11 11 8 10 10 11 13 13/30 Write block 372 len 4096 data: 9 14/30 Write block 382 len 20480 data: 17 19 17 13 12 15/30 Read block 528 len 36864 16/30 Read block 364 len 16384 17/30 Write block 76 len 36864 data: 9 10 10 12 11 12 12 15 14 18/30 Read block 35 len 40960 19/30 Read block 209 len 36864 20/30 Flush 21/30 Write block 234 len 12288 data: 9 8 8 22/30 Read block 571 len 24576 23/30 Read block 442 len 20480 24/30 Write block 211 len 40960 data: 12 11 15 14 13 11 12 13 12 9 25/30 Read block 145 len 24576 26/30 Read block 366 len 28672 27/30 Read block 174 len 24576 28/30 Read block 111 len 40960 29/30 Read block 331 len 4096 30/30 Read block 40 len 32768 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 9 10 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:41.632788212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 126 126 126 214 214 214 F F F 4 080-099 126 126 126 214 214 214 F F F 5 100-119 126 126 126 212 212 212 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 126 126 126 213 213 213 F F F 8 160-179 126 126 126 213 213 213 F F F 9 180-199 125 125 125 211 211 211 F F F 10 200-219 125 125 125 211 211 211 T T T 11 220-239 116 116 116 195 195 195 T T T 12 240-259 120 120 120 201 201 201 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 125 125 125 211 211 211 F F F 16 320-339 125 125 125 211 211 211 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 126 126 126 214 214 214 F F F 19 380-399 126 126 126 214 214 214 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 126 126 126 213 213 213 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 126 126 126 214 214 214 F F F 29 580-599 125 125 125 211 211 211 F F F Max gen: 126, Max flush: 214 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 126 126 126 214 214 214 F F F 4 080-099 126 126 126 214 214 214 F F F 5 100-119 126 126 126 212 212 212 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 126 126 126 213 213 213 F F F 8 160-179 126 126 126 213 213 213 F F F 9 180-199 125 125 125 211 211 211 F F F 10 200-219 127 127 127 215 215 215 F F F 11 220-239 127 127 127 215 215 215 F F F 12 240-259 120 120 120 201 201 201 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 125 125 125 211 211 211 F F F 16 320-339 125 125 125 211 211 211 F F F 17 340-359 118 118 118 197 197 197 F F F 18 360-379 126 126 126 214 214 214 F F F 19 380-399 126 126 126 214 214 214 F F F 20 400-419 118 118 118 198 198 198 F F F 21 420-439 126 126 126 213 213 213 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 126 126 126 214 214 214 F F F 29 580-599 125 125 125 211 211 211 F F F Max gen: 127, Max flush: 215 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.889316751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:45.932694538Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.933087607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.964862903Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.965481049Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.965523914Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.965540582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.965558271Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} {"msg":"Crucible 4caa182a-ec0d-4d7b-89d7-0803d6ec8d47 has session id: 20062063-5d7a-49cd-ae2f-a69b5bb71342","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966156178Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Upstairs opts: Upstairs UUID: 4caa182a-ec0d-4d7b-89d7-0803d6ec8d47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966197154Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Crucible stats registered with UUID: 4caa182a-ec0d-4d7b-89d7-0803d6ec8d47","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966213652Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:45.966226301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966687933Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.96671172Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} {"msg":"4caa182a-ec0d-4d7b-89d7-0803d6ec8d47 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966775224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966803571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"0","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966842607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"0","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966871654Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"1","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.96690242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"1","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966931417Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"2","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966958335Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"2","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.966984662Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967036926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967069093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"ds_connection connected from Ok(127.0.0.1:56631)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967189541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"ds_connection connected from Ok(127.0.0.1:60480)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967230646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"ds_connection connected from Ok(127.0.0.1:39879)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967255724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967774929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"2","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967804526Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"0","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.967820625Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"1","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.968474207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"1","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.968515142Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"2","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.968546369Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"0","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969084543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969192552Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[0]R flush_numbers[0..12]: [210, 211, 207, 214, 214, 212, 212, 213, 213, 211, 215, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969236967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[0]R generation[0..12]: [124, 125, 123, 126, 126, 126, 126, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969255585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969270454Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[1]R flush_numbers[0..12]: [210, 211, 207, 214, 214, 212, 212, 213, 213, 211, 215, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969289012Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[1]R generation[0..12]: [124, 125, 123, 126, 126, 126, 126, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969313499Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969340436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[2]R flush_numbers[0..12]: [210, 211, 207, 214, 214, 212, 212, 213, 213, 211, 215, 215]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969367314Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[2]R generation[0..12]: [124, 125, 123, 126, 126, 126, 126, 126, 126, 125, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969393991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:45.969419218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969443655Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969460294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969473463Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969486261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"":"downstairs","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.96949959Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969511659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"0","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969525167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"1","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969538456Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"client":"2","":"downstairs","session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969551784Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969567783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"4caa182a-ec0d-4d7b-89d7-0803d6ec8d47 is now active with session: 20062063-5d7a-49cd-ae2f-a69b5bb71342","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969581191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.96959323Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704,"session_id":"20062063-5d7a-49cd-ae2f-a69b5bb71342"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969605449Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} {"msg":"Activated sub_volume 4caa182a-ec0d-4d7b-89d7-0803d6ec8d47","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:45.969616677Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 375 len 4096 data: 9 03/30 Read block 435 len 20480 04/30 Read block 95 len 24576 05/30 Write block 230 len 20480 data: 10 14 13 11 10 06/30 Write block 326 len 36864 data: 8 8 9 10 8 9 8 6 6 07/30 Read block 535 len 32768 08/30 Write block 257 len 20480 data: 10 10 11 10 9 09/30 Read block 92 len 4096 10/30 Read block 56 len 24576 11/30 Write block 524 len 20480 data: 11 9 9 7 8 12/30 Write block 566 len 24576 data: 11 11 8 8 10 8 13/30 Write block 244 len 20480 data: 12 9 8 8 6 14/30 Read block 110 len 32768 15/30 Write block 404 len 28672 data: 13 10 10 5 5 10 11 16/30 Read block 19 len 24576 17/30 Read block 0 len 20480 18/30 Flush 19/30 Write block 352 len 16384 data: 12 14 12 14 20/30 Write block 189 len 36864 data: 10 11 12 11 10 12 11 12 10 21/30 Read block 139 len 8192 22/30 Read block 189 len 32768 23/30 Read block 336 len 32768 24/30 Write block 328 len 20480 data: 10 11 9 10 9 25/30 Read block 470 len 8192 26/30 Read block 295 len 36864 27/30 Write block 392 len 16384 data: 13 12 13 10 28/30 Write block 439 len 4096 data: 15 29/30 Write block 436 len 16384 data: 9 11 15 16 30/30 Write block 533 len 16384 data: 5 5 8 10 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:45.989281917Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 126 126 126 214 214 214 F F F 4 080-099 126 126 126 214 214 214 F F F 5 100-119 126 126 126 212 212 212 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 126 126 126 213 213 213 F F F 8 160-179 126 126 126 213 213 213 F F F 9 180-199 125 125 125 211 211 211 T T T 10 200-219 127 127 127 215 215 215 F F F 11 220-239 128 128 128 217 217 217 F F F 12 240-259 128 128 128 217 217 217 F F F 13 260-279 128 128 128 217 217 217 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 125 125 125 211 211 211 F F F 16 320-339 128 128 128 217 217 217 T T T 17 340-359 118 118 118 197 197 197 T T T 18 360-379 128 128 128 217 217 217 F F F 19 380-399 126 126 126 214 214 214 T T T 20 400-419 128 128 128 217 217 217 F F F 21 420-439 126 126 126 213 213 213 T T T 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 128 128 128 217 217 217 T T T 27 540-559 126 126 126 214 214 214 F F F 28 560-579 128 128 128 217 217 217 F F F 29 580-599 125 125 125 211 211 211 F F F Max gen: 128, Max flush: 217 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 125 125 125 211 211 211 F F F 2 040-059 123 123 123 207 207 207 F F F 3 060-079 126 126 126 214 214 214 F F F 4 080-099 126 126 126 214 214 214 F F F 5 100-119 126 126 126 212 212 212 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 126 126 126 213 213 213 F F F 8 160-179 126 126 126 213 213 213 F F F 9 180-199 129 129 129 218 218 218 F F F 10 200-219 127 127 127 215 215 215 F F F 11 220-239 128 128 128 217 217 217 F F F 12 240-259 128 128 128 217 217 217 F F F 13 260-279 128 128 128 217 217 217 F F F 14 280-299 122 122 122 204 204 204 F F F 15 300-319 125 125 125 211 211 211 F F F 16 320-339 129 129 129 218 218 218 F F F 17 340-359 129 129 129 218 218 218 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 129 129 129 218 218 218 F F F 20 400-419 128 128 128 217 217 217 F F F 21 420-439 129 129 129 218 218 218 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 119 119 119 200 200 200 F F F 26 520-539 129 129 129 218 218 218 F F F 27 540-559 126 126 126 214 214 214 F F F 28 560-579 128 128 128 217 217 217 F F F 29 580-599 125 125 125 211 211 211 F F F Max gen: 129, Max flush: 218 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.266912066Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.310761683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.311483948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.311519444Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.311534262Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.311548571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} {"msg":"Crucible 9674e3ed-e007-41e4-a25d-e9cef8b27ed1 has session id: 4af41f9b-0b6e-4bd5-848e-2f996091e04b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312191074Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Upstairs opts: Upstairs UUID: 9674e3ed-e007-41e4-a25d-e9cef8b27ed1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312237039Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Crucible stats registered with UUID: 9674e3ed-e007-41e4-a25d-e9cef8b27ed1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312260027Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:50.312279645Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312694492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312722269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} {"msg":"9674e3ed-e007-41e4-a25d-e9cef8b27ed1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312776993Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.31280562Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"0","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312836947Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"0","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312856015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"1","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.312960584Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"1","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313001249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"2","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313033716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"2","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313061403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.31308991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313118137Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56765)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313141485Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43169)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313165972Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48198)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313239725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313777399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"2","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.313805656Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"0","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.314444509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"0","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.314538759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"2","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.314988252Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.31510796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.315456113Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"1","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316034693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"1","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[0]R flush_numbers[0..12]: [210, 211, 207, 214, 214, 212, 212, 213, 213, 218, 215, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316565858Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[0]R generation[0..12]: [124, 125, 123, 126, 126, 126, 126, 126, 126, 129, 127, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316590995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316609053Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[1]R flush_numbers[0..12]: [210, 211, 207, 214, 214, 212, 212, 213, 213, 218, 215, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316626542Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[1]R generation[0..12]: [124, 125, 123, 126, 126, 126, 126, 126, 126, 129, 127, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.31664374Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316661078Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[2]R flush_numbers[0..12]: [210, 211, 207, 214, 214, 212, 212, 213, 213, 218, 215, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316677916Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[2]R generation[0..12]: [124, 125, 123, 126, 126, 126, 126, 126, 126, 129, 127, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316694934Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316712413Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:05:50.316728771Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316755998Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316772616Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316788565Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"":"downstairs","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316804773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316819481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"0","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.31683566Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"1","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316851808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"client":"2","":"downstairs","session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316868176Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316883375Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"9674e3ed-e007-41e4-a25d-e9cef8b27ed1 is now active with session: 4af41f9b-0b6e-4bd5-848e-2f996091e04b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316898983Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316913681Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712,"session_id":"4af41f9b-0b6e-4bd5-848e-2f996091e04b"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.31692853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} {"msg":"Activated sub_volume 9674e3ed-e007-41e4-a25d-e9cef8b27ed1","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:50.316942569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 37 len 4096 data: 13 03/30 Write block 523 len 16384 data: 11 12 10 10 04/30 Read block 364 len 36864 05/30 Read block 135 len 36864 06/30 Write block 89 len 20480 data: 13 13 12 14 11 07/30 Flush 08/30 Write block 159 len 32768 data: 13 14 10 9 9 9 7 11 09/30 Read block 82 len 32768 10/30 Read block 182 len 24576 11/30 Write block 299 len 28672 data: 15 13 14 17 17 16 13 12/30 Write block 551 len 8192 data: 7 7 13/30 Write block 254 len 8192 data: 8 9 14/30 Flush 15/30 Flush 16/30 Write block 521 len 20480 data: 7 9 12 13 11 17/30 Write block 193 len 28672 data: 11 13 12 13 11 9 10 18/30 Read block 485 len 24576 19/30 Read block 85 len 8192 20/30 Read block 551 len 24576 21/30 Read block 173 len 16384 22/30 Read block 213 len 36864 23/30 Read block 60 len 40960 24/30 Flush 25/30 Write block 500 len 32768 data: 10 11 13 11 10 8 9 7 26/30 Write block 573 len 12288 data: 13 14 14 27/30 Write block 260 len 32768 data: 11 10 6 6 8 10 10 10 28/30 Write block 44 len 40960 data: 11 8 7 6 4 5 8 8 10 7 29/30 Read block 212 len 12288 30/30 Write block 590 len 8192 data: 9 7 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:50.336484311Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 123 123 123 207 207 207 T T T 3 060-079 126 126 126 214 214 214 F F F 4 080-099 130 130 130 220 220 220 F F F 5 100-119 126 126 126 212 212 212 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 130 130 130 223 223 223 F F F 10 200-219 127 127 127 215 215 215 F F F 11 220-239 128 128 128 217 217 217 F F F 12 240-259 130 130 130 221 221 221 F F F 13 260-279 128 128 128 217 217 217 T T T 14 280-299 130 130 130 221 221 221 F F F 15 300-319 130 130 130 221 221 221 F F F 16 320-339 129 129 129 218 218 218 F F F 17 340-359 129 129 129 218 218 218 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 129 129 129 218 218 218 F F F 20 400-419 128 128 128 217 217 217 F F F 21 420-439 129 129 129 218 218 218 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 119 119 119 200 200 200 T T T 26 520-539 130 130 130 223 223 223 F F F 27 540-559 130 130 130 221 221 221 F F F 28 560-579 128 128 128 217 217 217 T T T 29 580-599 125 125 125 211 211 211 T T T Max gen: 130, Max flush: 223 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 131 131 131 224 224 224 F F F 3 060-079 126 126 126 214 214 214 F F F 4 080-099 130 130 130 220 220 220 F F F 5 100-119 126 126 126 212 212 212 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 130 130 130 223 223 223 F F F 10 200-219 127 127 127 215 215 215 F F F 11 220-239 128 128 128 217 217 217 F F F 12 240-259 130 130 130 221 221 221 F F F 13 260-279 131 131 131 224 224 224 F F F 14 280-299 130 130 130 221 221 221 F F F 15 300-319 130 130 130 221 221 221 F F F 16 320-339 129 129 129 218 218 218 F F F 17 340-359 129 129 129 218 218 218 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 129 129 129 218 218 218 F F F 20 400-419 128 128 128 217 217 217 F F F 21 420-439 129 129 129 218 218 218 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 130 130 130 223 223 223 F F F 27 540-559 130 130 130 221 221 221 F F F 28 560-579 131 131 131 224 224 224 F F F 29 580-599 131 131 131 224 224 224 F F F Max gen: 131, Max flush: 224 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.56990183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:54.612602197Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.612990827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.643730741Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.644304541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.644339118Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.644354056Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.644368535Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} {"msg":"Crucible 4d74cf1e-b0d2-4d41-a703-582c534c1abb has session id: 8a02d04b-5de5-4ac9-9ae1-5b8df5d69341","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645012167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Upstairs opts: Upstairs UUID: 4d74cf1e-b0d2-4d41-a703-582c534c1abb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645055253Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Crucible stats registered with UUID: 4d74cf1e-b0d2-4d41-a703-582c534c1abb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.64508351Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:54.645105198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645462651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645489668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} {"msg":"4d74cf1e-b0d2-4d41-a703-582c534c1abb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645540892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.64556854Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"0","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645625274Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"0","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.64565426Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"1","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645683997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"1","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645710575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"2","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645739322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"2","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645764199Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645790506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645820383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"ds_connection connected from Ok(127.0.0.1:62665)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645862049Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"ds_connection connected from Ok(127.0.0.1:36820)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645891386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"ds_connection connected from Ok(127.0.0.1:43707)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.645919413Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.646613751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"1","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.646696492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"2","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.646738487Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"0","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.647376411Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"2","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.647498888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"1","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.647527425Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"0","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.64796038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648091976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 224, 214, 220, 212, 212, 221, 221, 223, 215, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648218413Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[0]R generation[0..12]: [124, 130, 131, 126, 130, 126, 126, 130, 130, 130, 127, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648239111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648253829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 224, 214, 220, 212, 212, 221, 221, 223, 215, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648268458Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[1]R generation[0..12]: [124, 130, 131, 126, 130, 126, 126, 130, 130, 130, 127, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648285616Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648299895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 224, 214, 220, 212, 212, 221, 221, 223, 215, 217]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648313773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[2]R generation[0..12]: [124, 130, 131, 126, 130, 126, 126, 130, 130, 130, 127, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648327482Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.64834127Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648355149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648368168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Next flush: 225","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648388945Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648401954Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"":"downstairs","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648415053Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648426902Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"0","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.64844013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"1","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648453099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"client":"2","":"downstairs","session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648466027Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648479986Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"4d74cf1e-b0d2-4d41-a703-582c534c1abb is now active with session: 8a02d04b-5de5-4ac9-9ae1-5b8df5d69341","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648492795Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648504683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720,"session_id":"8a02d04b-5de5-4ac9-9ae1-5b8df5d69341"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648516782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} {"msg":"Activated sub_volume 4d74cf1e-b0d2-4d41-a703-582c534c1abb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:54.648527891Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 268 len 4096 data: 12 02/30 Read block 248 len 16384 03/30 Read block 447 len 36864 04/30 Read block 407 len 36864 05/30 Read block 423 len 36864 06/30 Flush 07/30 Write block 83 len 32768 data: 16 15 16 13 14 15 14 14 08/30 Write block 231 len 16384 data: 15 14 12 11 09/30 Write block 101 len 24576 data: 9 10 7 5 6 8 10/30 Write block 251 len 20480 data: 6 7 7 9 10 11/30 Write block 349 len 4096 data: 8 12/30 Write block 269 len 32768 data: 12 12 12 9 9 10 8 9 13/30 Write block 427 len 16384 data: 13 12 12 11 14/30 Flush 15/30 Write block 398 len 24576 data: 13 14 10 9 11 14 16/30 Read block 76 len 20480 17/30 Read block 22 len 4096 18/30 Write block 573 len 12288 data: 14 15 15 19/30 Flush 20/30 Flush 21/30 Write block 534 len 32768 data: 6 9 11 12 11 10 10 10 22/30 Write block 413 len 12288 data: 10 10 10 23/30 Read block 563 len 40960 24/30 Write block 184 len 40960 data: 11 10 11 15 13 11 12 13 12 12 25/30 Read block 504 len 32768 26/30 Read block 400 len 12288 27/30 Write block 248 len 8192 data: 7 7 28/30 Read block 152 len 24576 29/30 Write block 527 len 8192 data: 8 9 30/30 Read block 2 len 4096 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:54.668086451Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 131 131 131 224 224 224 F F F 3 060-079 126 126 126 214 214 214 F F F 4 080-099 132 132 132 226 226 226 F F F 5 100-119 132 132 132 226 226 226 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 130 130 130 223 223 223 T T T 10 200-219 127 127 127 215 215 215 F F F 11 220-239 132 132 132 226 226 226 F F F 12 240-259 132 132 132 226 226 226 T T T 13 260-279 132 132 132 226 226 226 F F F 14 280-299 130 130 130 221 221 221 F F F 15 300-319 130 130 130 221 221 221 F F F 16 320-339 129 129 129 218 218 218 F F F 17 340-359 132 132 132 226 226 226 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 132 132 132 227 227 227 T T T 21 420-439 132 132 132 226 226 226 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 130 130 130 223 223 223 T T T 27 540-559 130 130 130 221 221 221 T T T 28 560-579 132 132 132 227 227 227 F F F 29 580-599 131 131 131 224 224 224 F F F Max gen: 132, Max flush: 227 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 131 131 131 224 224 224 F F F 3 060-079 126 126 126 214 214 214 F F F 4 080-099 132 132 132 226 226 226 F F F 5 100-119 132 132 132 226 226 226 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 130 130 130 221 221 221 F F F 9 180-199 133 133 133 228 228 228 F F F 10 200-219 127 127 127 215 215 215 F F F 11 220-239 132 132 132 226 226 226 F F F 12 240-259 133 133 133 228 228 228 F F F 13 260-279 132 132 132 226 226 226 F F F 14 280-299 130 130 130 221 221 221 F F F 15 300-319 130 130 130 221 221 221 F F F 16 320-339 129 129 129 218 218 218 F F F 17 340-359 132 132 132 226 226 226 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 133 133 133 228 228 228 F F F 21 420-439 132 132 132 226 226 226 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 133 133 133 228 228 228 F F F 27 540-559 133 133 133 228 228 228 F F F 28 560-579 132 132 132 227 227 227 F F F 29 580-599 131 131 131 224 224 224 F F F Max gen: 133, Max flush: 228 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:58.933298176Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:58.976906168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:58.977313686Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.009132467Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.009739144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.00977687Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.009795018Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.009813116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} {"msg":"Crucible cbc07bb3-6cfd-486d-80d7-7d734186c1f5 has session id: c2ba3f9b-1453-4af1-861b-45bd8e9c4e62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010374398Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Upstairs opts: Upstairs UUID: cbc07bb3-6cfd-486d-80d7-7d734186c1f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010422663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Crucible stats registered with UUID: cbc07bb3-6cfd-486d-80d7-7d734186c1f5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01044416Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:59.010460399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010796734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010823771Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} {"msg":"cbc07bb3-6cfd-486d-80d7-7d734186c1f5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010894004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010912432Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"0","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010944688Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"0","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010959917Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"1","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010974345Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"1","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.010987414Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"2","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01101966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"2","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.011078804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01112403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.011156086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"ds_connection connected from Ok(127.0.0.1:43269)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.011186033Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"ds_connection connected from Ok(127.0.0.1:44633)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01121623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"ds_connection connected from Ok(127.0.0.1:46358)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.011243897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.011893239Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"1","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.011934145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"2","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.011964842Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"0","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.012481498Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"1","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01265277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"0","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.012787296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"2","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013060748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013245218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 224, 214, 226, 226, 212, 221, 221, 228, 215, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013361086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[0]R generation[0..12]: [124, 130, 131, 126, 132, 132, 126, 130, 130, 133, 127, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013380934Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013395703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 224, 214, 226, 226, 212, 221, 221, 228, 215, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013409501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[1]R generation[0..12]: [124, 130, 131, 126, 132, 132, 126, 130, 130, 133, 127, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01342294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013436848Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 224, 214, 226, 226, 212, 221, 221, 228, 215, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013450677Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[2]R generation[0..12]: [124, 130, 131, 126, 132, 132, 126, 130, 130, 133, 127, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013464145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013477964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013494182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013507791Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01352069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013540278Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"":"downstairs","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013553666Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013565885Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"0","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013579633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"1","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013592972Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"client":"2","":"downstairs","session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013606391Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.01361856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"cbc07bb3-6cfd-486d-80d7-7d734186c1f5 is now active with session: c2ba3f9b-1453-4af1-861b-45bd8e9c4e62","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013631438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013643217Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728,"session_id":"c2ba3f9b-1453-4af1-861b-45bd8e9c4e62"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013655276Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} {"msg":"Activated sub_volume cbc07bb3-6cfd-486d-80d7-7d734186c1f5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:05:59.013666244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 115 len 12288 data: 11 10 10 02/30 Flush 03/30 Write block 65 len 40960 data: 7 5 5 5 5 5 5 4 6 8 04/30 Flush 05/30 Write block 105 len 32768 data: 7 9 9 8 8 8 11 11 06/30 Read block 226 len 16384 07/30 Flush 08/30 Write block 79 len 8192 data: 13 12 09/30 Write block 545 len 40960 data: 11 12 11 9 7 7 8 8 8 11 10/30 Write block 557 len 24576 data: 11 11 12 12 12 10 11/30 Flush 12/30 Flush 13/30 Write block 244 len 28672 data: 13 10 9 9 8 8 5 14/30 Write block 178 len 32768 data: 14 11 8 10 10 9 12 11 15/30 Write block 58 len 28672 data: 15 17 15 13 11 9 7 16/30 Read block 408 len 16384 17/30 Write block 521 len 8192 data: 8 10 18/30 Read block 342 len 4096 19/30 Write block 342 len 4096 data: 10 20/30 Flush 21/30 Write block 74 len 28672 data: 9 9 10 11 11 14 13 22/30 Write block 213 len 8192 data: 16 15 23/30 Write block 471 len 12288 data: 5 6 6 24/30 Write block 334 len 32768 data: 7 5 6 8 9 9 8 8 25/30 Read block 315 len 12288 26/30 Read block 317 len 4096 27/30 Read block 446 len 8192 28/30 Write block 314 len 24576 data: 11 11 11 12 11 8 29/30 Read block 218 len 36864 30/30 Read block 236 len 8192 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:05:59.033531893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 134 134 134 234 234 234 F F F 3 060-079 134 134 134 234 234 234 T T T 4 080-099 134 134 134 232 232 232 T T T 5 100-119 134 134 134 231 231 231 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 134 134 134 234 234 234 F F F 9 180-199 134 134 134 234 234 234 F F F 10 200-219 127 127 127 215 215 215 T T T 11 220-239 132 132 132 226 226 226 F F F 12 240-259 134 134 134 234 234 234 F F F 13 260-279 132 132 132 226 226 226 F F F 14 280-299 130 130 130 221 221 221 F F F 15 300-319 130 130 130 221 221 221 T T T 16 320-339 129 129 129 218 218 218 T T T 17 340-359 134 134 134 234 234 234 T T T 18 360-379 128 128 128 217 217 217 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 133 133 133 228 228 228 F F F 21 420-439 132 132 132 226 226 226 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 125 125 125 211 211 211 T T T 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 134 134 134 232 232 232 F F F 29 580-599 131 131 131 224 224 224 F F F Max gen: 134, Max flush: 234 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 134 134 134 234 234 234 F F F 3 060-079 135 135 135 235 235 235 F F F 4 080-099 135 135 135 235 235 235 F F F 5 100-119 134 134 134 231 231 231 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 134 134 134 234 234 234 F F F 9 180-199 134 134 134 234 234 234 F F F 10 200-219 135 135 135 235 235 235 F F F 11 220-239 132 132 132 226 226 226 F F F 12 240-259 134 134 134 234 234 234 F F F 13 260-279 132 132 132 226 226 226 F F F 14 280-299 130 130 130 221 221 221 F F F 15 300-319 135 135 135 235 235 235 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 135 135 135 235 235 235 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 133 133 133 228 228 228 F F F 21 420-439 132 132 132 226 226 226 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 134 134 134 232 232 232 F F F 29 580-599 131 131 131 224 224 224 F F F Max gen: 135, Max flush: 235 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.314857207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:03.357663783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.358072961Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.38876596Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.389413782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.389448929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.389462987Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.389477875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} {"msg":"Crucible 3ef15fd2-a68c-44f0-ad79-0648a6c99627 has session id: e9f6a29c-df2d-482c-949a-baaabefcf3e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390027478Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Upstairs opts: Upstairs UUID: 3ef15fd2-a68c-44f0-ad79-0648a6c99627, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390062665Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Crucible stats registered with UUID: 3ef15fd2-a68c-44f0-ad79-0648a6c99627","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390077893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:03.390090402Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390529096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390553313Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} {"msg":"3ef15fd2-a68c-44f0-ad79-0648a6c99627 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390636125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390653543Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"0","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.39068167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"0","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390696288Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"1","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390711047Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"1","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390732325Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"2","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390759322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"2","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390783339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390809167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390842463Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37206)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.39087058Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57023)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390897827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60153)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.390922875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.391635561Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"1","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.391665617Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"2","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.391722292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"0","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.392221429Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"1","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.392368084Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"0","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.392497391Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"2","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.392743235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.392945804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 234, 235, 235, 231, 212, 221, 234, 234, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393103657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[0]R generation[0..12]: [124, 130, 134, 135, 135, 134, 126, 130, 134, 134, 135, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393125425Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393145313Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 234, 235, 235, 231, 212, 221, 234, 234, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393160271Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[1]R generation[0..12]: [124, 130, 134, 135, 135, 134, 126, 130, 134, 134, 135, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.39317445Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393188659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 234, 235, 235, 231, 212, 221, 234, 234, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393202317Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[2]R generation[0..12]: [124, 130, 134, 135, 135, 134, 126, 130, 134, 134, 135, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393216176Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393230214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393243023Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393256072Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.39326879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB"Wait for a query_work_queue command to finish before sending IO ,"pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393345022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"":"downstairs","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.39336145Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393373639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"0","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393387028Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"1","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393403186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"client":"2","":"downstairs","session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393417275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393429623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"3ef15fd2-a68c-44f0-ad79-0648a6c99627 is now active with session: e9f6a29c-df2d-482c-949a-baaabefcf3e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393442322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393454001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736,"session_id":"e9f6a29c-df2d-482c-949a-baaabefcf3e5"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.39346617Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} {"msg":"Activated sub_volume 3ef15fd2-a68c-44f0-ad79-0648a6c99627","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:03.393477458Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 63 len 16384 data: 10 8 8 6 02/30 Write block 188 len 4096 data: 14 03/30 Read block 152 len 32768 04/30 Write block 88 len 24576 data: 16 15 15 13 15 12 05/30 Write block 572 len 32768 data: 12 15 16 16 11 10 11 11 06/30 Write block 167 len 16384 data: 11 13 14 15 07/30 Flush 08/30 Write block 115 len 8192 data: 12 11 09/30 Read block 63 len 4096 10/30 Read block 409 len 16384 11/30 Read block 249 len 20480 12/30 Write block 52 len 36864 data: 11 8 9 12 10 12 16 18 16 13/30 Flush 14/30 Write block 578 len 36864 data: 12 12 6 7 6 6 8 10 9 15/30 Read block 544 len 20480 16/30 Read block 423 len 20480 17/30 Read block 390 len 16384 18/30 Write block 250 len 20480 data: 6 7 8 8 10 19/30 Read block 205 len 28672 20/30 Read block 509 len 4096 21/30 Read block 339 len 32768 22/30 Write block 76 len 24576 data: 11 12 12 15 14 13 23/30 Read block 504 len 4096 24/30 Write block 572 len 12288 data: 13 16 17 25/30 Read block 295 len 20480 26/30 Write block 77 len 28672 data: 13 13 16 15 14 13 17 27/30 Write block 75 len 24576 data: 10 12 14 14 17 16 28/30 Read block 186 len 20480 29/30 Read block 14 len 20480 30/30 Read block 550 len 36864 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 17 16 17 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:03.413653834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 136 136 136 237 237 237 F F F 3 060-079 136 136 136 237 237 237 T T T 4 080-099 136 136 136 236 236 236 T T T 5 100-119 136 136 136 237 237 237 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 136 136 136 236 236 236 F F F 10 200-219 135 135 135 235 235 235 F F F 11 220-239 132 132 132 226 226 226 F F F 12 240-259 134 134 134 234 234 234 T T T 13 260-279 132 132 132 226 226 226 F F F 14 280-299 130 130 130 221 221 221 F F F 15 300-319 135 135 135 235 235 235 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 135 135 135 235 235 235 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 133 133 133 228 228 228 F F F 21 420-439 132 132 132 226 226 226 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 136 136 136 236 236 236 T T T 29 580-599 131 131 131 224 224 224 T T T Max gen: 136, Max flush: 237 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 136 136 136 237 237 237 F F F 3 060-079 137 137 137 238 238 238 F F F 4 080-099 137 137 137 238 238 238 F F F 5 100-119 136 136 136 237 237 237 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 136 136 136 236 236 236 F F F 9 180-199 136 136 136 236 236 236 F F F 10 200-219 135 135 135 235 235 235 F F F 11 220-239 132 132 132 226 226 226 F F F 12 240-259 137 137 137 238 238 238 F F F 13 260-279 132 132 132 226 226 226 F F F 14 280-299 130 130 130 221 221 221 F F F 15 300-319 135 135 135 235 235 235 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 135 135 135 235 235 235 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 132 132 132 227 227 227 F F F 20 400-419 133 133 133 228 228 228 F F F 21 420-439 132 132 132 226 226 226 F F F 22 440-459 126 126 126 214 214 214 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 137 137 137 238 238 238 F F F 29 580-599 137 137 137 238 238 238 F F F Max gen: 137, Max flush: 238 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.682177005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.726194734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.726840277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.726884982Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.72690251Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.726920719Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} {"msg":"Crucible 0566ca6c-93fc-49b2-b003-aa419f863a79 has session id: 23e9aa12-8342-464d-aafa-7b7c31e49523","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.727502948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Upstairs opts: Upstairs UUID: 0566ca6c-93fc-49b2-b003-aa419f863a79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.727544114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Crucible stats registered with UUID: 0566ca6c-93fc-49b2-b003-aa419f863a79","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.727563492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:07.72757949Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728040092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728070629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} {"msg":"0566ca6c-93fc-49b2-b003-aa419f863a79 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728134362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.72815563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"0","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728184617Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"0","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728198735Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"1","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728212824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"1","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728226072Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"2","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728240481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"2","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728253649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728275927Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728297635Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"ds_connection connected from Ok(127.0.0.1:56588)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728518912Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"ds_connection connected from Ok(127.0.0.1:45107)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728560657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"ds_connection connected from Ok(127.0.0.1:59199)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.728589665Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.729142467Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"2","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.729282282Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"0","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.729564513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"1","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.729773101Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"2","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730021915Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"0","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730213335Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"1","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730427513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730498346Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 237, 238, 238, 237, 212, 221, 236, 236, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730783066Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[0]R generation[0..12]: [124, 130, 136, 137, 137, 136, 126, 130, 136, 136, 135, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730803574Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730818442Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 237, 238, 238, 237, 212, 221, 236, 236, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730832371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[1]R generation[0..12]: [124, 130, 136, 137, 137, 136, 126, 130, 136, 136, 135, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730845889Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730859888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 237, 238, 238, 237, 212, 221, 236, 236, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730873666Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[2]R generation[0..12]: [124, 130, 136, 137, 137, 136, 126, 130, 136, 136, 135, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730887305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730905343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730919142Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.73093265Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730945889Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730964597Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"":"downstairs","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730978365Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.730990504Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"0","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.731004283Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"1","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.731017811Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"client":"2","":"downstairs","session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.73103121Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.731043509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"0566ca6c-93fc-49b2-b003-aa419f863a79 is now active with session: 23e9aa12-8342-464d-aafa-7b7c31e49523","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.731056277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.731068206Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744,"session_id":"23e9aa12-8342-464d-aafa-7b7c31e49523"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.731080395Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} {"msg":"Activated sub_volume 0566ca6c-93fc-49b2-b003-aa419f863a79","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:07.731091504Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 12288 data: 9 9 11 02/30 Read block 224 len 20480 03/30 Write block 379 len 36864 data: 11 14 17 18 20 18 14 13 8 04/30 Read block 413 len 40960 05/30 Write block 232 len 20480 data: 15 13 12 9 9 06/30 Write block 412 len 28672 data: 10 11 11 11 9 11 12 07/30 Read block 469 len 20480 08/30 Read block 570 len 16384 09/30 Write block 366 len 32768 data: 9 10 7 9 11 8 10 7 10/30 Write block 80 len 36864 data: 17 15 14 18 16 17 14 15 17 11/30 Flush 12/30 Write block 440 len 8192 data: 12 9 13/30 Read block 179 len 36864 14/30 Read block 349 len 16384 15/30 Flush 16/30 Write block 173 len 36864 data: 13 11 14 18 18 15 12 9 11 17/30 Read block 574 len 20480 18/30 Read block 381 len 8192 19/30 Read block 152 len 28672 20/30 Write block 93 len 32768 data: 13 12 11 11 10 10 8 9 21/30 Write block 571 len 20480 data: 9 14 17 18 17 22/30 Read block 44 len 12288 23/30 Write block 350 len 16384 data: 10 10 13 15 24/30 Write block 194 len 24576 data: 14 13 14 12 10 11 25/30 Read block 175 len 8192 26/30 Read block 218 len 28672 27/30 Write block 594 len 20480 data: 5 4 3 3 3 28/30 Write block 277 len 16384 data: 10 8 9 12 29/30 Write block 235 len 36864 data: 10 10 9 9 12 10 14 15 14 30/30 Write block 201 len 36864 data: 12 11 8 10 10 11 10 10 10 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:07.752421719Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 136 136 136 237 237 237 F F F 3 060-079 137 137 137 238 238 238 F F F 4 080-099 138 138 138 239 239 239 T T T 5 100-119 136 136 136 237 237 237 T T T 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 136 136 136 236 236 236 T T T 9 180-199 136 136 136 236 236 236 T T T 10 200-219 135 135 135 235 235 235 T T T 11 220-239 138 138 138 239 239 239 T T T 12 240-259 137 137 137 238 238 238 T T T 13 260-279 132 132 132 226 226 226 T T T 14 280-299 130 130 130 221 221 221 T T T 15 300-319 135 135 135 235 235 235 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 138 138 138 239 239 239 T T T 18 360-379 138 138 138 239 239 239 F F F 19 380-399 138 138 138 239 239 239 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 132 132 132 226 226 226 F F F 22 440-459 138 138 138 240 240 240 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 137 137 137 238 238 238 T T T 29 580-599 137 137 137 238 238 238 T T T Max gen: 138, Max flush: 240 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 F F F 1 020-039 130 130 130 220 220 220 F F F 2 040-059 136 136 136 237 237 237 F F F 3 060-079 137 137 137 238 238 238 F F F 4 080-099 139 139 139 241 241 241 F F F 5 100-119 139 139 139 241 241 241 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 139 139 139 241 241 241 F F F 10 200-219 139 139 139 241 241 241 F F F 11 220-239 139 139 139 241 241 241 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 139 139 139 241 241 241 F F F 14 280-299 139 139 139 241 241 241 F F F 15 300-319 135 135 135 235 235 235 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 139 139 139 241 241 241 F F F 18 360-379 138 138 138 239 239 239 F F F 19 380-399 138 138 138 239 239 239 F F F 20 400-419 138 138 138 239 239 239 F F F 21 420-439 132 132 132 226 226 226 F F F 22 440-459 138 138 138 240 240 240 F F F 23 460-479 135 135 135 235 235 235 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 131 131 131 224 224 224 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 139 139 139 241 241 241 F F F 29 580-599 139 139 139 241 241 241 F F F Max gen: 139, Max flush: 241 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.025163168Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.069755848Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.070397401Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.070433487Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.070447745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.070465854Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} {"msg":"Crucible 60c11644-ede6-4cee-af3a-ecb611df75ff has session id: 81376626-9e97-4cd8-a2e6-d215398eee32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071037034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Upstairs opts: Upstairs UUID: 60c11644-ede6-4cee-af3a-ecb611df75ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.07107163Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Crucible stats registered with UUID: 60c11644-ede6-4cee-af3a-ecb611df75ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071086829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:12.071099967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071588726Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071633932Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} {"msg":"60c11644-ede6-4cee-af3a-ecb611df75ff active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071718943Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.07174597Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"0","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071777237Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"0","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071796205Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"1","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071823952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"1","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.07184364Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"2","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071858238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"2","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071875716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071895564Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.071922112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"ds_connection connected from Ok(127.0.0.1:51748)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.072099763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"ds_connection connected from Ok(127.0.0.1:60495)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.072133749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"ds_connection connected from Ok(127.0.0.1:55083)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.072158157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.072881262Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"2","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.072922657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"1","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.072954404Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"0","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.073526854Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"1","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.073556111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"2","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.073629164Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"0","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074161978Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074265737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 237, 238, 241, 241, 212, 221, 241, 241, 241, 241]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074340839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[0]R generation[0..12]: [124, 130, 136, 137, 139, 139, 126, 130, 139, 139, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074365537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074381585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 237, 238, 241, 241, 212, 221, 241, 241, 241, 241]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074395824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[1]R generation[0..12]: [124, 130, 136, 137, 139, 139, 126, 130, 139, 139, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074409642Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074423701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 237, 238, 241, 241, 212, 221, 241, 241, 241, 241]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074437349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[2]R generation[0..12]: [124, 130, 136, 137, 139, 139, 126, 130, 139, 139, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074450948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074464706Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074477435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074490744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074503553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074516371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"":"downstairs","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"No downstairs reconciliation required","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074536219Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074548208Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"0","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074563016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"1","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074576065Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"client":"2","":"downstairs","session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074588784Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074600882Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"60c11644-ede6-4cee-af3a-ecb611df75ff is now active with session: 81376626-9e97-4cd8-a2e6-d215398eee32","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074613721Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.0746254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752,"session_id":"81376626-9e97-4cd8-a2e6-d215398eee32"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074637639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} {"msg":"Activated sub_volume 60c11644-ede6-4cee-af3a-ecb611df75ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:12.074648647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 393 len 24576 data: 13 14 11 11 10 14 02/30 Write block 218 len 20480 data: 14 13 10 7 10 03/30 Read block 89 len 32768 04/30 Write block 75 len 16384 data: 11 13 15 15 05/30 Read block 300 len 4096 06/30 Read block 264 len 20480 07/30 Read block 571 len 32768 08/30 Read block 406 len 40960 09/30 Read block 59 len 16384 10/30 Write block 455 len 40960 data: 11 12 11 9 8 8 7 8 7 9 11/30 Flush 12/30 Write block 502 len 24576 data: 14 12 11 9 10 8 13/30 Write block 412 len 16384 data: 11 12 12 12 14/30 Flush 15/30 Write block 575 len 16384 data: 18 12 11 13 16/30 Read block 392 len 8192 17/30 Write block 268 len 20480 data: 13 13 13 13 10 18/30 Write block 376 len 12288 data: 7 9 10 19/30 Read block 278 len 4096 20/30 Write block 413 len 40960 data: 13 13 13 10 12 13 12 10 9 7 21/30 Flush 22/30 Write block 195 len 28672 data: 14 15 13 11 12 11 13 23/30 Read block 77 len 24576 24/30 Write block 100 len 8192 data: 10 10 25/30 Read block 85 len 24576 26/30 Read block 315 len 12288 27/30 Write block 441 len 40960 data: 10 7 8 9 12 12 12 9 11 11 28/30 Write block 13 len 32768 data: 12 14 14 14 14 13 14 11 29/30 Write block 269 len 24576 data: 14 14 14 11 10 11 30/30 Read block 151 len 36864 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 6 Sent Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 7 7 9 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:12.095685553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 210 210 210 T T T 1 020-039 130 130 130 220 220 220 T T T 2 040-059 136 136 136 237 237 237 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 139 139 139 241 241 241 F F F 5 100-119 139 139 139 241 241 241 T T T 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 139 139 139 241 241 241 T T T 10 200-219 140 140 140 242 242 242 T T T 11 220-239 140 140 140 242 242 242 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 140 140 140 244 244 244 T T T 14 280-299 139 139 139 241 241 241 F F F 15 300-319 135 135 135 235 235 235 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 139 139 139 241 241 241 F F F 18 360-379 140 140 140 244 244 244 F F F 19 380-399 140 140 140 242 242 242 F F F 20 400-419 140 140 140 244 244 244 F F F 21 420-439 140 140 140 244 244 244 F F F 22 440-459 140 140 140 242 242 242 T T T 23 460-479 140 140 140 242 242 242 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 140 140 140 243 243 243 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 140 140 140 244 244 244 F F F 29 580-599 139 139 139 241 241 241 F F F Max gen: 140, Max flush: 244 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 245 245 245 F F F 1 020-039 141 141 141 245 245 245 F F F 2 040-059 136 136 136 237 237 237 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 139 139 139 241 241 241 F F F 5 100-119 141 141 141 245 245 245 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 141 141 141 245 245 245 F F F 10 200-219 141 141 141 245 245 245 F F F 11 220-239 140 140 140 242 242 242 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 141 141 141 245 245 245 F F F 14 280-299 139 139 139 241 241 241 F F F 15 300-319 135 135 135 235 235 235 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 139 139 139 241 241 241 F F F 18 360-379 140 140 140 244 244 244 F F F 19 380-399 140 140 140 242 242 242 F F F 20 400-419 140 140 140 244 244 244 F F F 21 420-439 140 140 140 244 244 244 F F F 22 440-459 141 141 141 245 245 245 F F F 23 460-479 140 140 140 242 242 242 F F F 24 480-499 124 124 124 208 208 208 F F F 25 500-519 140 140 140 243 243 243 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 140 140 140 244 244 244 F F F 29 580-599 139 139 139 241 241 241 F F F Max gen: 141, Max flush: 245 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.346853992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.393793587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.394428481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.394464377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.394479346Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.394493714Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} {"msg":"Crucible 78446960-6800-45cb-a876-d8252cc0dc78 has session id: 20aade6b-8b0c-490f-8bfc-b6db248035f3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395098011Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Upstairs opts: Upstairs UUID: 78446960-6800-45cb-a876-d8252cc0dc78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395145306Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Crucible stats registered with UUID: 78446960-6800-45cb-a876-d8252cc0dc78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395170374Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:16.395191881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395606418Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395632015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} {"msg":"78446960-6800-45cb-a876-d8252cc0dc78 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395693079Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395716386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"0","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395744874Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"0","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395758512Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"1","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395772371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"1","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395785279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"2","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395799268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"2","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395812436Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395825925Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.395852622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53580)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.39597138Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38905)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.396014186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:61486)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.396043472Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.396727751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"1","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.396765087Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"0","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.397454315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"1","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.397622838Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"0","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.39807731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.398193978Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.3982708Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"2","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.398778007Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"2","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[0]R flush_numbers[0..12]: [245, 245, 237, 242, 241, 245, 212, 221, 241, 245, 245, 242]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399295123Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[0]R generation[0..12]: [141, 141, 136, 140, 139, 141, 126, 130, 139, 141, 141, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399316421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399331329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[1]R flush_numbers[0..12]: [245, 245, 237, 242, 241, 245, 212, 221, 241, 245, 245, 242]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399345198Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[1]R generation[0..12]: [141, 141, 136, 140, 139, 141, 126, 130, 139, 141, 141, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399358747Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399372375Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[2]R flush_numbers[0..12]: [245, 245, 237, 242, 241, 245, 212, 221, 241, 245, 245, 242]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399385984Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[2]R generation[0..12]: [141, 141, 136, 140, 139, 141, 126, 130, 139, 141, 141, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399399652Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.39942068Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:06:16.399445577Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399472455Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399487063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399500092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"":"downstairs","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.39951708Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399539098Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"0","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399564265Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"1","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399589513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"client":"2","":"downstairs","session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.39961435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399632828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"78446960-6800-45cb-a876-d8252cc0dc78 is now active with session: 20aade6b-8b0c-490f-8bfc-b6db248035f3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399652146Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:06:16.399673964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760,"session_id":"20aade6b-8b0c-490f-8bfc-b6db248035f3"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399701921Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} {"msg":"Activated sub_volume 78446960-6800-45cb-a876-d8252cc0dc78","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:16.399717509Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 279 len 4096 data: 10 02/30 Read block 153 len 28672 03/30 Read block 532 len 32768 04/30 Write block 310 len 40960 data: 13 13 13 13 12 12 12 13 12 9 05/30 Write block 472 len 28672 data: 7 7 6 7 8 12 11 06/30 Flush 07/30 Write block 424 len 16384 data: 9 11 11 14 08/30 Write block 224 len 16384 data: 7 6 5 6 09/30 Flush 10/30 Write block 400 len 24576 data: 11 10 12 15 14 11 11/30 Write block 278 len 8192 data: 9 11 12/30 Write block 13 len 28672 data: 13 15 15 15 15 14 15 13/30 Read block 578 len 12288 14/30 Write block 52 len 4096 data: 12 15/30 Read block 27 len 20480 16/30 Flush 17/30 Write block 425 len 16384 data: 12 12 15 13 18/30 Read block 181 len 4096 19/30 Write block 50 len 20480 data: 9 9 13 9 10 20/30 Write block 30 len 8192 data: 12 11 21/30 Read block 5 len 32768 22/30 Write block 358 len 8192 data: 13 9 23/30 Write block 562 len 16384 data: 11 11 11 11 24/30 Write block 46 len 16384 data: 8 7 5 6 25/30 Write block 193 len 32768 data: 13 15 15 16 14 12 13 12 26/30 Write block 40 len 8192 data: 13 14 27/30 Write block 230 len 32768 data: 11 16 16 14 13 11 11 10 28/30 Write block 361 len 20480 data: 9 8 7 8 9 29/30 Write block 467 len 20480 data: 7 7 6 5 6 30/30 Write block 479 len 32768 data: 12 10 10 8 8 8 10 10 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:16.416905697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 248 248 248 F F F 1 020-039 141 141 141 245 245 245 T T T 2 040-059 142 142 142 248 248 248 T T T 3 060-079 140 140 140 242 242 242 F F F 4 080-099 139 139 139 241 241 241 F F F 5 100-119 141 141 141 245 245 245 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 141 141 141 245 245 245 T T T 10 200-219 141 141 141 245 245 245 T T T 11 220-239 142 142 142 247 247 247 T T T 12 240-259 139 139 139 241 241 241 F F F 13 260-279 142 142 142 248 248 248 F F F 14 280-299 139 139 139 241 241 241 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 139 139 139 241 241 241 T T T 18 360-379 140 140 140 244 244 244 T T T 19 380-399 140 140 140 242 242 242 F F F 20 400-419 142 142 142 248 248 248 F F F 21 420-439 142 142 142 247 247 247 T T T 22 440-459 141 141 141 245 245 245 F F F 23 460-479 142 142 142 246 246 246 T T T 24 480-499 124 124 124 208 208 208 T T T 25 500-519 140 140 140 243 243 243 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 140 140 140 244 244 244 T T T 29 580-599 139 139 139 241 241 241 F F F Max gen: 142, Max flush: 248 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 248 248 248 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 143 143 143 249 249 249 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 139 139 139 241 241 241 F F F 5 100-119 141 141 141 245 245 245 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 143 143 143 249 249 249 F F F 10 200-219 143 143 143 249 249 249 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 142 142 142 248 248 248 F F F 14 280-299 139 139 139 241 241 241 F F F 15 300-319 142 142 142 246 246 246 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 143 143 143 249 249 249 F F F 19 380-399 140 140 140 242 242 242 F F F 20 400-419 142 142 142 248 248 248 F F F 21 420-439 143 143 143 249 249 249 F F F 22 440-459 141 141 141 245 245 245 F F F 23 460-479 143 143 143 249 249 249 F F F 24 480-499 143 143 143 249 249 249 F F F 25 500-519 140 140 140 243 243 243 F F F 26 520-539 134 134 134 234 234 234 F F F 27 540-559 134 134 134 232 232 232 F F F 28 560-579 143 143 143 249 249 249 F F F 29 580-599 139 139 139 241 241 241 F F F Max gen: 143, Max flush: 249 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.6867846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:30.732230401Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.73262077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76445909Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765040639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765075945Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765095854Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76512875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} {"msg":"Crucible 24d7bdd7-d976-4ee9-8b17-00dbca8f4d6c has session id: 019d0ec0-e54b-4d43-ab7b-42dfb257b18f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765630888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Upstairs opts: Upstairs UUID: 24d7bdd7-d976-4ee9-8b17-00dbca8f4d6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765688812Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Crucible stats registered with UUID: 24d7bdd7-d976-4ee9-8b17-00dbca8f4d6c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.765708339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:30.765722328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766193339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766218236Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} {"msg":"24d7bdd7-d976-4ee9-8b17-00dbca8f4d6c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766292809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766310507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"0","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766339014Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"0","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766353402Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"1","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766367981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"1","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766381149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"2","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766398727Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"2","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766412006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766425495Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766439403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55816)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766515385Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39478)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766548002Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52459)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.766576159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.76733623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"1","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.767540619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"0","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.767580764Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"2","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768043036Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"1","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768180242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"2","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768362473Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"0","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768628045Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768716816Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[0]R flush_numbers[0..12]: [248, 249, 249, 242, 241, 245, 212, 221, 241, 249, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768934183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[0]R generation[0..12]: [142, 143, 143, 140, 139, 141, 126, 130, 139, 143, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768954391Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768969099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[1]R flush_numbers[0..12]: [248, 249, 249, 242, 241, 245, 212, 221, 241, 249, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768983678Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[1]R generation[0..12]: [142, 143, 143, 140, 139, 141, 126, 130, 139, 143, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.768997507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769011415Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[2]R flush_numbers[0..12]: [248, 249, 249, 242, 241, 245, 212, 221, 241, 249, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769025174Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[2]R generation[0..12]: [142, 143, 143, 140, 139, 141, 126, 130, 139, 143, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769039162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769053171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769066229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769079248Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769091977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769127553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"":"downstairs","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769147681Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769168589Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"0","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769182227Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"1","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769195276Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"client":"2","":"downstairs","session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769208314Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769220503Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"24d7bdd7-d976-4ee9-8b17-00dbca8f4d6c is now active with session: 019d0ec0-e54b-4d43-ab7b-42dfb257b18f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769233192Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769245151Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768,"session_id":"019d0ec0-e54b-4d43-ab7b-42dfb257b18f"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769257109Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} {"msg":"Activated sub_volume 24d7bdd7-d976-4ee9-8b17-00dbca8f4d6c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:30.769268098Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 596 len 12288 data: 4 4 4 02/30 Flush 03/30 Write block 523 len 12288 data: 13 14 12 04/30 Write block 48 len 24576 data: 6 7 10 10 14 10 05/30 Write block 453 len 40960 data: 13 11 12 13 12 10 9 9 8 9 06/30 Read block 67 len 16384 07/30 Write block 409 len 4096 data: 11 08/30 Flush 09/30 Write block 81 len 12288 data: 16 15 19 10/30 Read block 392 len 8192 11/30 Read block 184 len 16384 12/30 Flush 13/30 Write block 262 len 4096 data: 7 14/30 Write block 88 len 20480 data: 18 16 16 14 16 15/30 Read block 53 len 4096 16/30 Flush 17/30 Write block 282 len 28672 data: 12 14 15 13 15 17 16 18/30 Write block 516 len 12288 data: 7 7 6 19/30 Write block 107 len 24576 data: 10 9 9 9 12 12 20/30 Read block 60 len 12288 21/30 Write block 196 len 8192 data: 17 15 22/30 Write block 473 len 40960 data: 8 7 8 9 13 12 13 11 11 9 23/30 Write block 562 len 36864 data: 12 12 12 12 12 12 9 9 11 24/30 Read block 120 len 4096 25/30 Read block 567 len 16384 26/30 Write block 307 len 4096 data: 10 27/30 Write block 398 len 16384 data: 15 15 12 11 28/30 Write block 543 len 36864 data: 7 8 12 13 12 10 8 8 9 29/30 Read block 156 len 40960 30/30 Read block 322 len 4096 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:30.787454491Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1768} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 248 248 248 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 144 144 144 251 251 251 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 144 144 144 253 253 253 F F F 5 100-119 141 141 141 245 245 245 T T T 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 143 143 143 249 249 249 T T T 10 200-219 143 143 143 249 249 249 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 144 144 144 253 253 253 F F F 14 280-299 139 139 139 241 241 241 T T T 15 300-319 142 142 142 246 246 246 T T T 16 320-339 135 135 135 235 235 235 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 143 143 143 249 249 249 F F F 19 380-399 140 140 140 242 242 242 T T T 20 400-419 144 144 144 251 251 251 T T T 21 420-439 143 143 143 249 249 249 F F F 22 440-459 144 144 144 251 251 251 F F F 23 460-479 144 144 144 251 251 251 T T T 24 480-499 143 143 143 249 249 249 T T T 25 500-519 140 140 140 243 243 243 T T T 26 520-539 144 144 144 251 251 251 F F F 27 540-559 134 134 134 232 232 232 T T T 28 560-579 143 143 143 249 249 249 T T T 29 580-599 144 144 144 250 250 250 F F F Max gen: 144, Max flush: 253 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 248 248 248 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 144 144 144 251 251 251 F F F 3 060-079 140 140 140 242 242 242 F F F 4 080-099 144 144 144 253 253 253 F F F 5 100-119 145 145 145 254 254 254 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 139 139 139 241 241 241 F F F 9 180-199 145 145 145 254 254 254 F F F 10 200-219 143 143 143 249 249 249 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 144 144 144 253 253 253 F F F 14 280-299 145 145 145 254 254 254 F F F 15 300-319 145 145 145 254 254 254 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 143 143 143 249 249 249 F F F 18 360-379 143 143 143 249 249 249 F F F 19 380-399 145 145 145 254 254 254 F F F 20 400-419 145 145 145 254 254 254 F F F 21 420-439 143 143 143 249 249 249 F F F 22 440-459 144 144 144 251 251 251 F F F 23 460-479 145 145 145 254 254 254 F F F 24 480-499 145 145 145 254 254 254 F F F 25 500-519 145 145 145 254 254 254 F F F 26 520-539 144 144 144 251 251 251 F F F 27 540-559 145 145 145 254 254 254 F F F 28 560-579 145 145 145 254 254 254 F F F 29 580-599 144 144 144 250 250 250 F F F Max gen: 145, Max flush: 254 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.085538518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.130642544Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.131245072Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.131282508Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.131297376Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.131311875Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} {"msg":"Crucible 0a096976-da9f-427b-aeb8-038ffd35eb5e has session id: 7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.13193792Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Upstairs opts: Upstairs UUID: 0a096976-da9f-427b-aeb8-038ffd35eb5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.131981355Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Crucible stats registered with UUID: 0a096976-da9f-427b-aeb8-038ffd35eb5e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132006792Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:35.13203224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132563004Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132596621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} {"msg":"0a096976-da9f-427b-aeb8-038ffd35eb5e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132654225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132681502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"0","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132712729Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"0","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132726937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"1","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132740956Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"1","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132754164Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"2","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132770553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"2","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.13279414Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132820697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132848265Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45978)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132875152Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58690)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132902299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:52346)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.132929356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.133489018Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"0","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.133649351Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"1","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.133879927Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"2","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.134100464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"0","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.134257588Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"1","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.134573855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"2","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.134853805Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.134971943Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[0]R flush_numbers[0..12]: [248, 249, 251, 242, 253, 254, 212, 221, 241, 254, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135104879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[0]R generation[0..12]: [142, 143, 144, 140, 144, 145, 126, 130, 139, 145, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135127067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135141925Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[1]R flush_numbers[0..12]: [248, 249, 251, 242, 253, 254, 212, 221, 241, 254, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135156954Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[1]R generation[0..12]: [142, 143, 144, 140, 144, 145, 126, 130, 139, 145, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135170932Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135185081Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[2]R flush_numbers[0..12]: [248, 249, 251, 242, 253, 254, 212, 221, 241, 254, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135201979Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[2]R generation[0..12]: [142, 143, 144, 140, 144, 145, 126, 130, 139, 145, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135215988Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135230216Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135243315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135256263Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135269032Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.13528865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"":"downstairs","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135302119Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135313948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"0","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135328136Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"1","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135341294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"client":"2","":"downstairs","session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135354553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135366952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"0a096976-da9f-427b-aeb8-038ffd35eb5e is now active with session: 7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.13537985Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135391759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776,"session_id":"7d76c0cc-11cf-4ae9-8de2-6d76221bc0fb"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135406658Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} {"msg":"Activated sub_volume 0a096976-da9f-427b-aeb8-038ffd35eb5e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:35.135417897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 32768 data: 9 8 7 8 11 11 15 11 02/30 Write block 534 len 4096 data: 7 03/30 Write block 279 len 12288 data: 12 13 12 04/30 Write block 171 len 32768 data: 17 11 14 12 15 19 19 16 05/30 Flush 06/30 Write block 340 len 24576 data: 10 10 12 9 8 7 07/30 Read block 304 len 4096 08/30 Read block 42 len 32768 09/30 Write block 6 len 36864 data: 12 12 9 10 9 12 13 14 16 10/30 Read block 266 len 40960 11/30 Read block 25 len 8192 12/30 Write block 449 len 32768 data: 12 12 12 14 14 12 13 14 13/30 Write block 371 len 36864 data: 9 11 8 6 10 8 10 11 12 14/30 Read block 288 len 24576 15/30 Read block 172 len 8192 16/30 Read block 446 len 16384 17/30 Write block 593 len 28672 data: 7 6 5 5 5 5 3 18/30 Write block 348 len 16384 data: 10 9 11 11 19/30 Flush 20/30 Write block 401 len 20480 data: 12 13 16 15 12 21/30 Write block 396 len 36864 data: 12 11 16 16 13 13 14 17 16 22/30 Write block 291 len 12288 data: 18 16 13 23/30 Read block 588 len 24576 24/30 Read block 366 len 4096 25/30 Write block 72 len 20480 data: 5 7 10 12 14 26/30 Read block 430 len 28672 27/30 Read block 560 len 20480 28/30 Read block 174 len 16384 29/30 Write block 461 len 32768 data: 9 10 8 10 7 6 8 8 30/30 Write block 400 len 16384 data: 14 14 15 18 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:35.155647347Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1776} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 256 256 256 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 146 146 146 255 255 255 F F F 3 060-079 140 140 140 242 242 242 T T T 4 080-099 144 144 144 253 253 253 F F F 5 100-119 145 145 145 254 254 254 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 146 146 146 255 255 255 F F F 9 180-199 145 145 145 254 254 254 F F F 10 200-219 143 143 143 249 249 249 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 146 146 146 255 255 255 F F F 14 280-299 146 146 146 255 255 255 T T T 15 300-319 145 145 145 254 254 254 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 146 146 146 256 256 256 F F F 18 360-379 146 146 146 256 256 256 F F F 19 380-399 145 145 145 254 254 254 T T T 20 400-419 145 145 145 254 254 254 T T T 21 420-439 143 143 143 249 249 249 F F F 22 440-459 146 146 146 256 256 256 F F F 23 460-479 145 145 145 254 254 254 T T T 24 480-499 145 145 145 254 254 254 F F F 25 500-519 145 145 145 254 254 254 F F F 26 520-539 146 146 146 255 255 255 F F F 27 540-559 145 145 145 254 254 254 F F F 28 560-579 145 145 145 254 254 254 F F F 29 580-599 146 146 146 256 256 256 F F F Max gen: 146, Max flush: 256 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 256 256 256 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 146 146 146 255 255 255 F F F 3 060-079 147 147 147 257 257 257 F F F 4 080-099 144 144 144 253 253 253 F F F 5 100-119 145 145 145 254 254 254 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 146 146 146 255 255 255 F F F 9 180-199 145 145 145 254 254 254 F F F 10 200-219 143 143 143 249 249 249 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 146 146 146 255 255 255 F F F 14 280-299 147 147 147 257 257 257 F F F 15 300-319 145 145 145 254 254 254 F F F 16 320-339 135 135 135 235 235 235 F F F 17 340-359 146 146 146 256 256 256 F F F 18 360-379 146 146 146 256 256 256 F F F 19 380-399 147 147 147 257 257 257 F F F 20 400-419 147 147 147 257 257 257 F F F 21 420-439 143 143 143 249 249 249 F F F 22 440-459 146 146 146 256 256 256 F F F 23 460-479 147 147 147 257 257 257 F F F 24 480-499 145 145 145 254 254 254 F F F 25 500-519 145 145 145 254 254 254 F F F 26 520-539 146 146 146 255 255 255 F F F 27 540-559 145 145 145 254 254 254 F F F 28 560-579 145 145 145 254 254 254 F F F 29 580-599 146 146 146 256 256 256 F F F Max gen: 147, Max flush: 257 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.397514846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:39.441256205Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.441627706Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.473075916Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.473648207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.473682183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.473696042Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.4737102Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} {"msg":"Crucible dcd98582-ca11-41ee-b85b-2822c27f2f5b has session id: 010b5506-c59a-4ec3-88f0-75d464191729","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474164143Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Upstairs opts: Upstairs UUID: dcd98582-ca11-41ee-b85b-2822c27f2f5b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474208058Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Crucible stats registered with UUID: dcd98582-ca11-41ee-b85b-2822c27f2f5b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474230726Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:39.474248724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.47466734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474688498Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} {"msg":"dcd98582-ca11-41ee-b85b-2822c27f2f5b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.47476221Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474779109Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"0","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474806866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"0","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474820514Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"1","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474838672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"1","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474852771Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"2","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.47486692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"2","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474880038Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474896496Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.474921664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"ds_connection connected from Ok(127.0.0.1:57553)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.47495918Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"ds_connection connected from Ok(127.0.0.1:63799)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.475174477Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"ds_connection connected from Ok(127.0.0.1:46693)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.475216853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.475660037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"1","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.475788803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"2","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.475899022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"0","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.476389301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"1","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.476439096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"2","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.476474612Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"0","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477002287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477106396Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[0]R flush_numbers[0..12]: [256, 249, 255, 257, 253, 254, 212, 221, 255, 254, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477175579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[0]R generation[0..12]: [146, 143, 146, 147, 144, 145, 126, 130, 146, 145, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477195637Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477210005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[1]R flush_numbers[0..12]: [256, 249, 255, 257, 253, 254, 212, 221, 255, 254, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477223744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[1]R generation[0..12]: [146, 143, 146, 147, 144, 145, 126, 130, 146, 145, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477237492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477251511Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[2]R flush_numbers[0..12]: [256, 249, 255, 257, 253, 254, 212, 221, 255, 254, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477265309Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[2]R generation[0..12]: [146, 143, 146, 147, 144, 145, 126, 130, 146, 145, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477279128Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477293367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477306165Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477319244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477336832Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477350061Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"":"downstairs","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:39.477363599Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477380277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"0","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477393566Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"1","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477406595Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"client":"2","":"downstairs","session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477419823Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477432022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"dcd98582-ca11-41ee-b85b-2822c27f2f5b is now active with session: 010b5506-c59a-4ec3-88f0-75d464191729","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477444691Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477456519Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784,"session_id":"010b5506-c59a-4ec3-88f0-75d464191729"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477468588Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} {"msg":"Activated sub_volume dcd98582-ca11-41ee-b85b-2822c27f2f5b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:39.477479587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 473 len 40960 data: 9 8 9 10 14 13 14 12 12 10 02/30 Write block 324 len 20480 data: 11 10 9 9 11 03/30 Flush 04/30 Flush 05/30 Write block 183 len 16384 data: 10 13 12 12 06/30 Read block 121 len 12288 07/30 Read block 191 len 20480 08/30 Read block 359 len 24576 09/30 Read block 399 len 32768 10/30 Write block 91 len 40960 data: 15 17 14 13 12 12 11 11 9 11 11/30 Read block 243 len 4096 12/30 Write block 194 len 12288 data: 16 16 18 13/30 Write block 444 len 28672 data: 10 13 13 13 10 13 13 14/30 Flush 15/30 Write block 563 len 24576 data: 13 13 13 13 13 10 16/30 Write block 505 len 24576 data: 10 11 9 7 9 12 17/30 Write block 304 len 12288 data: 17 14 11 18/30 Read block 96 len 8192 19/30 Read block 18 len 20480 20/30 Read block 374 len 28672 21/30 Read block 31 len 16384 22/30 Read block 274 len 32768 23/30 Read block 519 len 24576 24/30 Read block 2 len 32768 25/30 Write block 82 len 36864 data: 16 20 17 18 15 16 19 17 17 26/30 Flush 27/30 Write block 396 len 24576 data: 13 12 17 17 15 15 28/30 Write block 473 len 28672 data: 10 9 10 11 15 14 15 29/30 Read block 217 len 32768 30/30 Read block 207 len 28672 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:39.499342907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 256 256 256 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 146 146 146 255 255 255 F F F 3 060-079 147 147 147 257 257 257 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 148 148 148 260 260 260 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 146 146 146 255 255 255 F F F 9 180-199 148 148 148 260 260 260 F F F 10 200-219 143 143 143 249 249 249 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 146 146 146 255 255 255 F F F 14 280-299 147 147 147 257 257 257 F F F 15 300-319 148 148 148 261 261 261 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 146 146 146 256 256 256 F F F 18 360-379 146 146 146 256 256 256 F F F 19 380-399 147 147 147 257 257 257 T T T 20 400-419 147 147 147 257 257 257 T T T 21 420-439 143 143 143 249 249 249 F F F 22 440-459 148 148 148 260 260 260 F F F 23 460-479 148 148 148 258 258 258 T T T 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 261 261 261 F F F 26 520-539 146 146 146 255 255 255 F F F 27 540-559 145 145 145 254 254 254 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 146 146 146 256 256 256 F F F Max gen: 148, Max flush: 261 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 256 256 256 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 146 146 146 255 255 255 F F F 3 060-079 147 147 147 257 257 257 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 148 148 148 260 260 260 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 F F F 8 160-179 146 146 146 255 255 255 F F F 9 180-199 148 148 148 260 260 260 F F F 10 200-219 143 143 143 249 249 249 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 146 146 146 255 255 255 F F F 14 280-299 147 147 147 257 257 257 F F F 15 300-319 148 148 148 261 261 261 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 146 146 146 256 256 256 F F F 18 360-379 146 146 146 256 256 256 F F F 19 380-399 149 149 149 262 262 262 F F F 20 400-419 149 149 149 262 262 262 F F F 21 420-439 143 143 143 249 249 249 F F F 22 440-459 148 148 148 260 260 260 F F F 23 460-479 149 149 149 262 262 262 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 261 261 261 F F F 26 520-539 146 146 146 255 255 255 F F F 27 540-559 145 145 145 254 254 254 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 146 146 146 256 256 256 F F F Max gen: 149, Max flush: 262 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.757235724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:43.800128761Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.800546128Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.831181503Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.831790289Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.831824586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.831838764Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.831852783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} {"msg":"Crucible ad95993d-2c04-4efa-ad24-e7aa35c6bf1e has session id: 2097b895-18f5-4273-933b-c321d6679fad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.832490107Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Upstairs opts: Upstairs UUID: ad95993d-2c04-4efa-ad24-e7aa35c6bf1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.832534892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Crucible stats registered with UUID: ad95993d-2c04-4efa-ad24-e7aa35c6bf1e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.832559899Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:43.832577627Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.8329395Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.832963787Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} {"msg":"ad95993d-2c04-4efa-ad24-e7aa35c6bf1e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.83303257Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833054318Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"0","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833090414Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"0","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833108172Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"1","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.83312623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"1","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833142639Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"2","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833160007Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"2","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833176185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833193423Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833210231Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"ds_connection connected from Ok(127.0.0.1:50474)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833255567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"ds_connection connected from Ok(127.0.0.1:38101)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.833285164Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"ds_connection connected from Ok(127.0.0.1:43826)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.83331768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.834073061Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"1","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.834101748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"2","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.834117717Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"0","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.834681808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"1","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.834795206Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"0","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.83495459Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"2","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835362867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835481804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[0]R flush_numbers[0..12]: [256, 249, 255, 257, 261, 260, 212, 221, 255, 260, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835549328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[0]R generation[0..12]: [146, 143, 146, 147, 148, 148, 126, 130, 146, 148, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835568586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835583204Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[1]R flush_numbers[0..12]: [256, 249, 255, 257, 261, 260, 212, 221, 255, 260, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835597012Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[1]R generation[0..12]: [146, 143, 146, 147, 148, 148, 126, 130, 146, 148, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835610881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835629419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[2]R flush_numbers[0..12]: [256, 249, 255, 257, 261, 260, 212, 221, 255, 260, 249, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835643827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[2]R generation[0..12]: [146, 143, 146, 147, 148, 148, 126, 130, 146, 148, 143, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835657616Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835671435Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835684183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835697322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835710001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835722759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"":"downstairs","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835735688Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:43.835747627Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"0","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835764325Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"1","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835777294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"client":"2","":"downstairs","session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835790362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835802521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"ad95993d-2c04-4efa-ad24-e7aa35c6bf1e is now active with session: 2097b895-18f5-4273-933b-c321d6679fad","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.83581514Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835829188Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792,"session_id":"2097b895-18f5-4273-933b-c321d6679fad"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835841477Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} {"msg":"Activated sub_volume ad95993d-2c04-4efa-ad24-e7aa35c6bf1e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:43.835852606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 206 len 4096 data: 12 02/30 Read block 174 len 32768 03/30 Write block 52 len 32768 data: 16 12 11 13 11 13 17 19 04/30 Read block 480 len 4096 05/30 Flush 06/30 Write block 291 len 20480 data: 19 17 14 14 14 07/30 Write block 189 len 4096 data: 12 08/30 Write block 41 len 8192 data: 15 12 09/30 Read block 462 len 40960 10/30 Read block 465 len 24576 11/30 Write block 378 len 28672 data: 12 13 15 18 19 21 19 12/30 Read block 368 len 12288 13/30 Read block 518 len 36864 14/30 Write block 363 len 20480 data: 8 9 10 10 11 15/30 Read block 279 len 24576 16/30 Read block 76 len 28672 17/30 Flush 18/30 Write block 107 len 24576 data: 11 10 10 10 13 13 19/30 Write block 304 len 36864 data: 18 15 12 11 11 11 14 14 14 20/30 Write block 434 len 36864 data: 11 9 10 12 16 17 13 11 8 21/30 Read block 105 len 20480 22/30 Write block 4 len 8192 data: 9 9 23/30 Write block 272 len 24576 data: 12 11 12 9 10 11 24/30 Read block 474 len 16384 25/30 Write block 292 len 32768 data: 18 15 15 15 14 15 15 16 26/30 Write block 522 len 4096 data: 11 27/30 Write block 152 len 40960 data: 9 10 10 10 9 11 13 14 15 11 28/30 Write block 192 len 4096 data: 13 29/30 Write block 592 len 12288 data: 6 8 7 30/30 Write block 456 len 24576 data: 15 13 11 10 10 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 10 Done Sent Sent false 1027 Acked Write 1 Done Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 4 4 10 Done 11 9 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:43.855129466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1792} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 256 256 256 T T T 1 020-039 143 143 143 249 249 249 F F F 2 040-059 150 150 150 264 264 264 F F F 3 060-079 147 147 147 257 257 257 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 148 148 148 260 260 260 T T T 6 120-139 126 126 126 212 212 212 F F F 7 140-159 130 130 130 221 221 221 T T T 8 160-179 146 146 146 255 255 255 T T T 9 180-199 150 150 150 264 264 264 T T T 10 200-219 150 150 150 263 263 263 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 146 146 146 255 255 255 T T T 14 280-299 150 150 150 264 264 264 T T T 15 300-319 148 148 148 261 261 261 T T T 16 320-339 148 148 148 258 258 258 F F F 17 340-359 146 146 146 256 256 256 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 149 149 149 262 262 262 F F F 21 420-439 143 143 143 249 249 249 T T T 22 440-459 148 148 148 260 260 260 T T T 23 460-479 149 149 149 262 262 262 T T T 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 261 261 261 F F F 26 520-539 146 146 146 255 255 255 T T T 27 540-559 145 145 145 254 254 254 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 146 146 146 256 256 256 T T T Max gen: 150, Max flush: 264 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 F F F 1 020-039 143 143 143 249 249 249 F F F 2 040-059 150 150 150 264 264 264 F F F 3 060-079 147 147 147 257 257 257 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 151 151 151 265 265 265 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 151 151 151 265 265 265 F F F 8 160-179 151 151 151 265 265 265 F F F 9 180-199 151 151 151 265 265 265 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 139 139 139 241 241 241 F F F 13 260-279 151 151 151 265 265 265 F F F 14 280-299 151 151 151 265 265 265 F F F 15 300-319 151 151 151 265 265 265 F F F 16 320-339 148 148 148 258 258 258 F F F 17 340-359 146 146 146 256 256 256 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 149 149 149 262 262 262 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 151 151 151 265 265 265 F F F 23 460-479 151 151 151 265 265 265 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 148 148 148 261 261 261 F F F 26 520-539 151 151 151 265 265 265 F F F 27 540-559 145 145 145 254 254 254 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 151 151 151 265 265 265 F F F Max gen: 151, Max flush: 265 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.162228972Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.206759369Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.207391183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.207433738Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.207450667Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.207466295Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} {"msg":"Crucible 9f12ed75-c776-4bec-afcc-56cdbe463cf8 has session id: 7fa74106-8d3f-4cda-afef-a1de6fa9f08e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208114907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Upstairs opts: Upstairs UUID: 9f12ed75-c776-4bec-afcc-56cdbe463cf8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208161322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Crucible stats registered with UUID: 9f12ed75-c776-4bec-afcc-56cdbe463cf8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.2081846Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:48.208198599Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208671809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208696947Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} {"msg":"9f12ed75-c776-4bec-afcc-56cdbe463cf8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208783768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208802216Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"0","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208831013Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"0","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208845571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"1","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208863219Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"1","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208876938Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"2","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208891157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"2","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208904125Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208917554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.208931052Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36966)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.209244609Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56283)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.209318672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50984)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.209346729Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.209896612Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"2","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.209959405Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"0","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.210133987Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"1","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.210657382Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"2","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.210688489Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"0","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.210718616Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"1","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211261879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211371798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[0]R flush_numbers[0..12]: [265, 249, 264, 257, 261, 265, 212, 265, 265, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211438961Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[0]R generation[0..12]: [151, 143, 150, 147, 148, 151, 126, 151, 151, 151, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211467238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211482836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[1]R flush_numbers[0..12]: [265, 249, 264, 257, 261, 265, 212, 265, 265, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211496925Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[1]R generation[0..12]: [151, 143, 150, 147, 148, 151, 126, 151, 151, 151, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211510623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211524282Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[2]R flush_numbers[0..12]: [265, 249, 264, 257, 261, 265, 212, 265, 265, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.21153822Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[2]R generation[0..12]: [151, 143, 150, 147, 148, 151, 126, 151, 151, 151, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211551659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211566078Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:48.211579136Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211596844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211613813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211627111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"":"downstairs","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.21164028Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211652149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"0","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211665247Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"1","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211677966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"client":"2","":"downstairs","session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211690774Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211702733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"9f12ed75-c776-4bec-afcc-56cdbe463cf8 is now active with session: 7fa74106-8d3f-4cda-afef-a1de6fa9f08e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211715322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211726901Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800,"session_id":"7fa74106-8d3f-4cda-afef-a1de6fa9f08e"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.2117389Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} {"msg":"Activated sub_volume 9f12ed75-c776-4bec-afcc-56cdbe463cf8","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:48.211749998Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 40960 data: 10 8 8 8 7 7 8 9 12 14 02/30 Flush 03/30 Write block 34 len 40960 data: 14 13 12 14 14 15 14 16 13 12 04/30 Read block 228 len 40960 05/30 Read block 413 len 8192 06/30 Flush 07/30 Write block 163 len 16384 data: 10 10 8 12 08/30 Read block 563 len 28672 09/30 Read block 285 len 40960 10/30 Write block 330 len 8192 data: 10 11 11/30 Read block 295 len 28672 12/30 Write block 257 len 12288 data: 11 11 12 13/30 Write block 288 len 4096 data: 17 14/30 Flush 15/30 Write block 352 len 8192 data: 14 16 16/30 Read block 161 len 24576 17/30 Write block 287 len 24576 data: 18 18 16 18 20 19 18/30 Read block 571 len 8192 19/30 Write block 278 len 32768 data: 10 13 14 13 13 15 16 14 20/30 Write block 280 len 28672 data: 15 14 14 16 17 15 16 21/30 Read block 59 len 28672 22/30 Write block 410 len 20480 data: 12 11 12 14 14 23/30 Flush 24/30 Write block 309 len 12288 data: 12 15 15 25/30 Read block 241 len 20480 26/30 Write block 47 len 24576 data: 9 8 9 12 12 17 27/30 Write block 552 len 4096 data: 9 28/30 Read block 440 len 24576 29/30 Read block 191 len 40960 30/30 Read block 589 len 8192 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 7 6 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:48.232408524Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1800} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 F F F 1 020-039 152 152 152 267 267 267 F F F 2 040-059 152 152 152 267 267 267 T T T 3 060-079 147 147 147 257 257 257 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 151 151 151 265 265 265 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 151 151 151 265 265 265 F F F 8 160-179 152 152 152 268 268 268 F F F 9 180-199 151 151 151 265 265 265 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 152 152 152 269 269 269 F F F 15 300-319 151 151 151 265 265 265 T T T 16 320-339 152 152 152 268 268 268 F F F 17 340-359 152 152 152 269 269 269 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 152 152 152 269 269 269 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 151 151 151 265 265 265 F F F 23 460-479 151 151 151 265 265 265 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 152 152 152 266 266 266 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 145 145 145 254 254 254 T T T 28 560-579 148 148 148 261 261 261 F F F 29 580-599 151 151 151 265 265 265 F F F Max gen: 152, Max flush: 269 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 F F F 1 020-039 152 152 152 267 267 267 F F F 2 040-059 153 153 153 270 270 270 F F F 3 060-079 147 147 147 257 257 257 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 151 151 151 265 265 265 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 151 151 151 265 265 265 F F F 8 160-179 152 152 152 268 268 268 F F F 9 180-199 151 151 151 265 265 265 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 152 152 152 269 269 269 F F F 15 300-319 153 153 153 270 270 270 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 152 152 152 269 269 269 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 152 152 152 269 269 269 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 151 151 151 265 265 265 F F F 23 460-479 151 151 151 265 265 265 F F F 24 480-499 148 148 148 258 258 258 F F F 25 500-519 152 152 152 266 266 266 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 153 153 153 270 270 270 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 151 151 151 265 265 265 F F F Max gen: 153, Max flush: 270 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.45810051Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:52.500696048Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.501108965Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.531767588Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.532341298Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.532383184Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.532399992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.53241749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} {"msg":"Crucible 16d4812e-a153-428e-88cf-ba1b438842cd has session id: 5b0274b3-3066-4e8c-84f5-791c38ad9a19","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533009728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Upstairs opts: Upstairs UUID: 16d4812e-a153-428e-88cf-ba1b438842cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533056534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Crucible stats registered with UUID: 16d4812e-a153-428e-88cf-ba1b438842cd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533085071Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:52.533106578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533480749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533505367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} {"msg":"16d4812e-a153-428e-88cf-ba1b438842cd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.53356976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533587448Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"0","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533615395Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"0","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533629144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"1","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533648492Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"1","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533673249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"2","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533698087Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"2","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533712565Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533736093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.5337636Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"ds_connection connected from Ok(127.0.0.1:43043)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533791947Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"ds_connection connected from Ok(127.0.0.1:49410)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533844761Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"ds_connection connected from Ok(127.0.0.1:54134)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.533979567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.534664066Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"1","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.534701922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"0","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.534732199Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"2","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.535377112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"1","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.535494219Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"0","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.535556383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"2","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536006766Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536112025Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[0]R flush_numbers[0..12]: [265, 267, 270, 257, 261, 265, 212, 265, 268, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536167259Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[0]R generation[0..12]: [151, 152, 153, 147, 148, 151, 126, 151, 152, 151, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536186277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536200976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[1]R flush_numbers[0..12]: [265, 267, 270, 257, 261, 265, 212, 265, 268, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536214764Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[1]R generation[0..12]: [151, 152, 153, 147, 148, 151, 126, 151, 152, 151, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536232172Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536246711Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[2]R flush_numbers[0..12]: [265, 267, 270, 257, 261, 265, 212, 265, 268, 265, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536261699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[2]R generation[0..12]: [151, 152, 153, 147, 148, 151, 126, 151, 152, 151, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536275508Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536289386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536302155Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:06:52.536315154Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536381697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536398305Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"":"downstairs","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536411884Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536423902Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"0","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536437291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"1","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.53645034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"client":"2","":"downstairs","session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536463428Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536475607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"16d4812e-a153-428e-88cf-ba1b438842cd is now active with session: 5b0274b3-3066-4e8c-84f5-791c38ad9a19","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536492425Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536504494Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808,"session_id":"5b0274b3-3066-4e8c-84f5-791c38ad9a19"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536516593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} {"msg":"Activated sub_volume 16d4812e-a153-428e-88cf-ba1b438842cd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:52.536527501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 44 len 8192 data: 12 9 03/30 Write block 33 len 40960 data: 13 15 14 13 15 15 16 15 17 14 04/30 Write block 140 len 20480 data: 10 10 8 9 10 05/30 Flush 06/30 Write block 158 len 24576 data: 14 15 16 12 10 11 07/30 Read block 282 len 36864 08/30 Read block 402 len 24576 09/30 Read block 269 len 8192 10/30 Read block 401 len 32768 11/30 Write block 156 len 20480 data: 10 12 15 16 17 12/30 Read block 312 len 36864 13/30 Read block 73 len 20480 14/30 Read block 423 len 8192 15/30 Write block 498 len 32768 data: 11 11 11 12 15 13 12 11 16/30 Flush 17/30 Write block 71 len 12288 data: 6 6 8 18/30 Read block 597 len 4096 19/30 Read block 470 len 4096 20/30 Read block 480 len 4096 21/30 Read block 107 len 32768 22/30 Read block 327 len 40960 23/30 Write block 297 len 16384 data: 16 16 17 14 24/30 Write block 186 len 24576 data: 13 16 15 13 13 14 25/30 Read block 83 len 40960 26/30 Write block 352 len 12288 data: 15 17 13 27/30 Read block 540 len 32768 28/30 Write block 555 len 4096 data: 11 29/30 Read block 314 len 20480 30/30 Read block 371 len 36864 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 14 14 13 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:52.556727755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1808} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 F F F 1 020-039 154 154 154 272 272 272 F F F 2 040-059 154 154 154 272 272 272 F F F 3 060-079 147 147 147 257 257 257 T T T 4 080-099 148 148 148 261 261 261 F F F 5 100-119 151 151 151 265 265 265 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 154 154 154 273 273 273 F F F 8 160-179 154 154 154 273 273 273 F F F 9 180-199 151 151 151 265 265 265 T T T 10 200-219 150 150 150 263 263 263 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 152 152 152 269 269 269 T T T 15 300-319 153 153 153 270 270 270 T T T 16 320-339 152 152 152 268 268 268 F F F 17 340-359 152 152 152 269 269 269 T T T 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 152 152 152 269 269 269 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 151 151 151 265 265 265 F F F 23 460-479 151 151 151 265 265 265 F F F 24 480-499 154 154 154 273 273 273 F F F 25 500-519 154 154 154 273 273 273 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 153 153 153 270 270 270 T T T 28 560-579 148 148 148 261 261 261 F F F 29 580-599 151 151 151 265 265 265 F F F Max gen: 154, Max flush: 273 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 F F F 1 020-039 154 154 154 272 272 272 F F F 2 040-059 154 154 154 272 272 272 F F F 3 060-079 155 155 155 274 274 274 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 151 151 151 265 265 265 F F F 6 120-139 126 126 126 212 212 212 F F F 7 140-159 154 154 154 273 273 273 F F F 8 160-179 154 154 154 273 273 273 F F F 9 180-199 155 155 155 274 274 274 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 143 143 143 249 249 249 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 155 155 155 274 274 274 F F F 15 300-319 155 155 155 274 274 274 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 152 152 152 269 269 269 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 151 151 151 265 265 265 F F F 23 460-479 151 151 151 265 265 265 F F F 24 480-499 154 154 154 273 273 273 F F F 25 500-519 154 154 154 273 273 273 F F F 26 520-539 152 152 152 266 266 266 F F F 27 540-559 155 155 155 274 274 274 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 151 151 151 265 265 265 F F F Max gen: 155, Max flush: 274 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.8346931Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.885162577Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.88580169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.885843416Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.885858224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.885872693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} {"msg":"Crucible 0f978c42-3314-40d2-83e6-645b760da83b has session id: ef533797-704f-48fc-a64a-204e45197083","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.886463981Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Upstairs opts: Upstairs UUID: 0f978c42-3314-40d2-83e6-645b760da83b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.886576159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Crucible stats registered with UUID: 0f978c42-3314-40d2-83e6-645b760da83b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.886612455Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:56.886634703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887102415Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887129812Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} {"msg":"0f978c42-3314-40d2-83e6-645b760da83b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887190255Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887216792Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"0","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887247239Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"0","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887262388Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"1","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887277326Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"1","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887295724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"2","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887312153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"2","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.88733569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887360928Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887385235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"ds_connection connected from Ok(127.0.0.1:46741)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887412042Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"ds_connection connected from Ok(127.0.0.1:50593)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887439679Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"ds_connection connected from Ok(127.0.0.1:61090)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.887673495Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.888138696Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"0","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.888320717Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"1","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.888427116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"2","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.888838134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"0","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.888879929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"1","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.88915673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"2","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889433851Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889533921Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[0]R flush_numbers[0..12]: [265, 272, 272, 274, 261, 265, 212, 273, 273, 274, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889753578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[0]R generation[0..12]: [151, 154, 154, 155, 148, 151, 126, 154, 154, 155, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889773606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889789144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[1]R flush_numbers[0..12]: [265, 272, 272, 274, 261, 265, 212, 273, 273, 274, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889803413Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[1]R generation[0..12]: [151, 154, 154, 155, 148, 151, 126, 154, 154, 155, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889817171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.88983103Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[2]R flush_numbers[0..12]: [265, 272, 272, 274, 261, 265, 212, 273, 273, 274, 263, 249]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889844829Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[2]R generation[0..12]: [151, 154, 154, 155, 148, 151, 126, 154, 154, 155, 150, 143]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889858397Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889872026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889884774Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889897903Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.8899226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2025-03-18T05:06:56.889940128Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"":"downstairs","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889961226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889973185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"0","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889986324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"1","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.889999292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"client":"2","":"downstairs","session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.890012871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.8900253Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"0f978c42-3314-40d2-83e6-645b760da83b is now active with session: ef533797-704f-48fc-a64a-204e45197083","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.890038158Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.890050037Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816,"session_id":"ef533797-704f-48fc-a64a-204e45197083"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.890062076Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} {"msg":"Activated sub_volume 0f978c42-3314-40d2-83e6-645b760da83b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:06:56.890073045Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 63 len 24576 data: 11 9 9 7 6 6 03/30 Read block 154 len 12288 04/30 Write block 115 len 40960 data: 13 12 11 12 15 13 10 10 9 10 05/30 Write block 583 len 36864 data: 7 9 11 10 11 9 11 10 8 06/30 Read block 295 len 4096 07/30 Write block 106 len 12288 data: 10 12 11 08/30 Read block 506 len 36864 09/30 Read block 142 len 40960 10/30 Write block 184 len 40960 data: 14 13 14 17 16 14 14 15 14 14 11/30 Read block 351 len 24576 12/30 Write block 305 len 24576 data: 16 13 12 12 13 16 13/30 Write block 313 len 20480 data: 14 13 13 13 14 14/30 Write block 224 len 4096 data: 8 15/30 Write block 555 len 20480 data: 12 10 12 12 13 16/30 Read block 388 len 8192 17/30 Read block 208 len 24576 18/30 Write block 288 len 36864 data: 19 17 19 21 20 16 16 16 15 19/30 Read block 355 len 12288 20/30 Read block 425 len 24576 21/30 Read block 499 len 36864 22/30 Write block 134 len 8192 data: 9 14 23/30 Write block 119 len 8192 data: 16 14 24/30 Read block 257 len 36864 25/30 Read block 216 len 12288 26/30 Read block 139 len 28672 27/30 Write block 41 len 4096 data: 18 28/30 Write block 401 len 40960 data: 16 16 19 17 13 11 6 6 12 13 29/30 Write block 522 len 28672 data: 13 15 15 13 11 9 10 30/30 Read block 81 len 32768 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 29 27 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:06:56.910699554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1816} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 F F F 1 020-039 154 154 154 272 272 272 F F F 2 040-059 154 154 154 272 272 272 T T T 3 060-079 155 155 155 274 274 274 T T T 4 080-099 148 148 148 261 261 261 F F F 5 100-119 151 151 151 265 265 265 T T T 6 120-139 126 126 126 212 212 212 T T T 7 140-159 154 154 154 273 273 273 F F F 8 160-179 154 154 154 273 273 273 F F F 9 180-199 155 155 155 274 274 274 T T T 10 200-219 150 150 150 263 263 263 F F F 11 220-239 143 143 143 249 249 249 T T T 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 155 155 155 274 274 274 T T T 15 300-319 155 155 155 274 274 274 T T T 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 152 152 152 269 269 269 T T T 21 420-439 151 151 151 265 265 265 F F F 22 440-459 151 151 151 265 265 265 F F F 23 460-479 151 151 151 265 265 265 F F F 24 480-499 154 154 154 273 273 273 F F F 25 500-519 154 154 154 273 273 273 F F F 26 520-539 152 152 152 266 266 266 T T T 27 540-559 155 155 155 274 274 274 T T T 28 560-579 148 148 148 261 261 261 F F F 29 580-599 151 151 151 265 265 265 T T T Max gen: 155, Max flush: 274 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 F F F 1 020-039 154 154 154 272 272 272 F F F 2 040-059 156 156 156 275 275 275 F F F 3 060-079 156 156 156 275 275 275 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 156 156 156 275 275 275 F F F 6 120-139 156 156 156 275 275 275 F F F 7 140-159 154 154 154 273 273 273 F F F 8 160-179 154 154 154 273 273 273 F F F 9 180-199 156 156 156 275 275 275 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 156 156 156 275 275 275 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 156 156 156 275 275 275 F F F 15 300-319 156 156 156 275 275 275 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 150 150 150 264 264 264 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 151 151 151 265 265 265 F F F 23 460-479 151 151 151 265 265 265 F F F 24 480-499 154 154 154 273 273 273 F F F 25 500-519 154 154 154 273 273 273 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 156 156 156 275 275 275 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 156 156 156 275 275 275 F F F Max gen: 156, Max flush: 275 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.185415122Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.229552919Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230182183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23021865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230232728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230247237Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} {"msg":"Crucible 6295a762-4457-48c4-a050-b7039412d1fa has session id: 066f0f18-a8f2-49aa-ac48-21fe07bbe127","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230750664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Upstairs opts: Upstairs UUID: 6295a762-4457-48c4-a050-b7039412d1fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23079453Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Crucible stats registered with UUID: 6295a762-4457-48c4-a050-b7039412d1fa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.230821337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:11.230844154Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231314995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231350292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} {"msg":"6295a762-4457-48c4-a050-b7039412d1fa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231442182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231470219Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"0","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231500226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"0","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231515734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"1","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231530273Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"1","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231543811Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"2","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23155822Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"2","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231571649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231590827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231613434Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"ds_connection connected from Ok(127.0.0.1:51627)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23174663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"ds_connection connected from Ok(127.0.0.1:39193)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231786836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"ds_connection connected from Ok(127.0.0.1:61897)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.231813153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.232474334Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"0","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23251162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"1","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23318508Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"1","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.233225276Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"0","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.233775739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.233895126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.233924613Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"2","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.234442399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"2","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[0]R flush_numbers[0..12]: [265, 272, 275, 275, 261, 275, 275, 273, 273, 275, 263, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.234953466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[0]R generation[0..12]: [151, 154, 156, 156, 148, 156, 156, 154, 154, 156, 150, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.234973464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.234992042Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[1]R flush_numbers[0..12]: [265, 272, 275, 275, 261, 275, 275, 273, 273, 275, 263, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.2350065Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[1]R generation[0..12]: [151, 154, 156, 156, 148, 156, 156, 154, 154, 156, 150, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235020499Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235034427Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[2]R flush_numbers[0..12]: [265, 272, 275, 275, 261, 275, 275, 273, 273, 275, 263, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235048106Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[2]R generation[0..12]: [151, 154, 156, 156, 148, 156, 156, 154, 154, 156, 150, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235061854Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235075493Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235088342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23510141Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235114269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:07:11.235127018Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"":"downstairs","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235146716Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235158515Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"0","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235171553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"1","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235186641Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"client":"2","":"downstairs","session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.23519976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235212099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"6295a762-4457-48c4-a050-b7039412d1fa is now active with session: 066f0f18-a8f2-49aa-ac48-21fe07bbe127","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235224868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235236876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824,"session_id":"066f0f18-a8f2-49aa-ac48-21fe07bbe127"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235248975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} {"msg":"Activated sub_volume 6295a762-4457-48c4-a050-b7039412d1fa","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:11.235260034Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 32768 data: 20 22 21 17 17 17 16 17 02/30 Flush 03/30 Write block 502 len 24576 data: 16 14 13 12 12 10 04/30 Read block 387 len 40960 05/30 Write block 380 len 16384 data: 16 19 20 22 06/30 Write block 24 len 4096 data: 8 07/30 Read block 274 len 16384 08/30 Read block 567 len 16384 09/30 Read block 296 len 24576 10/30 Write block 442 len 8192 data: 9 9 11/30 Read block 196 len 32768 12/30 Read block 309 len 20480 13/30 Read block 570 len 24576 14/30 Read block 148 len 12288 15/30 Read block 482 len 20480 16/30 Write block 510 len 36864 data: 13 13 12 9 11 9 9 9 8 17/30 Read block 242 len 28672 18/30 Flush 19/30 Flush 20/30 Flush 21/30 Write block 181 len 28672 data: 12 11 11 15 14 15 18 22/30 Read block 237 len 12288 23/30 Read block 299 len 4096 24/30 Write block 476 len 36864 data: 12 16 15 16 13 13 11 9 9 25/30 Read block 398 len 24576 26/30 Write block 314 len 16384 data: 14 14 14 15 27/30 Read block 181 len 20480 28/30 Read block 32 len 24576 29/30 Write block 0 len 36864 data: 5 7 9 8 10 10 13 13 10 30/30 Write block 137 len 24576 data: 11 10 11 11 11 9 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:11.256483751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1824} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 265 265 265 T T T 1 020-039 158 158 158 277 277 277 F F F 2 040-059 156 156 156 275 275 275 F F F 3 060-079 156 156 156 275 275 275 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 156 156 156 275 275 275 F F F 6 120-139 156 156 156 275 275 275 T T T 7 140-159 154 154 154 273 273 273 T T T 8 160-179 154 154 154 273 273 273 F F F 9 180-199 156 156 156 275 275 275 T T T 10 200-219 150 150 150 263 263 263 F F F 11 220-239 156 156 156 275 275 275 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 156 156 156 275 275 275 T T T 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 158 158 158 277 277 277 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 158 158 158 277 277 277 F F F 23 460-479 151 151 151 265 265 265 T T T 24 480-499 154 154 154 273 273 273 T T T 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 156 156 156 275 275 275 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 156 156 156 275 275 275 F F F Max gen: 158, Max flush: 277 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 278 278 278 F F F 1 020-039 158 158 158 277 277 277 F F F 2 040-059 156 156 156 275 275 275 F F F 3 060-079 156 156 156 275 275 275 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 156 156 156 275 275 275 F F F 6 120-139 159 159 159 278 278 278 F F F 7 140-159 159 159 159 278 278 278 F F F 8 160-179 154 154 154 273 273 273 F F F 9 180-199 159 159 159 278 278 278 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 156 156 156 275 275 275 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 159 159 159 278 278 278 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 264 264 264 F F F 19 380-399 158 158 158 277 277 277 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 151 151 151 265 265 265 F F F 22 440-459 158 158 158 277 277 277 F F F 23 460-479 159 159 159 278 278 278 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 156 156 156 275 275 275 F F F 28 560-579 148 148 148 261 261 261 F F F 29 580-599 156 156 156 275 275 275 F F F Max gen: 159, Max flush: 278 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.504463643Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:15.547570398Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.547960207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.579632744Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.580243171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.580282727Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.580297645Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.580311864Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} {"msg":"Crucible 24cd11b6-7e57-4985-93c9-53a67c249b42 has session id: d85e33ec-dcd7-47f2-b09c-03f9c6cd715f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.580862046Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Upstairs opts: Upstairs UUID: 24cd11b6-7e57-4985-93c9-53a67c249b42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.580905612Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Crucible stats registered with UUID: 24cd11b6-7e57-4985-93c9-53a67c249b42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.58092521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:15.580938668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581373963Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.58139835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} {"msg":"24cd11b6-7e57-4985-93c9-53a67c249b42 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581453934Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581485821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"0","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581515528Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"0","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581530446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"1","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581544865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"1","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581557844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"2","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581571692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"2","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581584521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581600519Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581620517Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33702)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581679621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35805)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.581721217Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50285)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.58187879Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.582346121Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"1","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.582386377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"2","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.582546171Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"0","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583058207Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"1","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583088254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"2","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583142868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"0","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583709219Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583734557Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[0]R flush_numbers[0..12]: [278, 277, 275, 275, 261, 275, 278, 278, 273, 278, 263, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.5838001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[0]R generation[0..12]: [159, 158, 156, 156, 148, 156, 159, 159, 154, 159, 150, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583820958Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583835726Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[1]R flush_numbers[0..12]: [278, 277, 275, 275, 261, 275, 278, 278, 273, 278, 263, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583849855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[1]R generation[0..12]: [159, 158, 156, 156, 148, 156, 159, 159, 154, 159, 150, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583863623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583877442Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[2]R flush_numbers[0..12]: [278, 277, 275, 275, 261, 275, 278, 278, 273, 278, 263, 275]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.58389105Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[2]R generation[0..12]: [159, 158, 156, 156, 148, 156, 159, 159, 154, 159, 150, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583904589Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583921907Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583935086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583947934Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.583960633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:07:15.583973342Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"":"downstairs","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.58399177Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584004048Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"0","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584017847Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"1","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584031096Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"client":"2","":"downstairs","session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584044524Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584056963Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"24cd11b6-7e57-4985-93c9-53a67c249b42 is now active with session: d85e33ec-dcd7-47f2-b09c-03f9c6cd715f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584069682Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584081571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832,"session_id":"d85e33ec-dcd7-47f2-b09c-03f9c6cd715f"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584093629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} {"msg":"Activated sub_volume 24cd11b6-7e57-4985-93c9-53a67c249b42","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:15.584104578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 437 len 28672 data: 13 17 18 14 12 10 10 02/30 Write block 438 len 40960 data: 18 19 15 13 11 11 11 14 14 14 03/30 Read block 321 len 24576 04/30 Read block 480 len 28672 05/30 Read block 37 len 36864 06/30 Write block 157 len 40960 data: 13 16 17 18 13 11 12 11 9 13 07/30 Read block 455 len 4096 08/30 Read block 167 len 24576 09/30 Write block 572 len 40960 data: 15 18 19 19 13 12 14 13 7 8 10/30 Read block 162 len 36864 11/30 Flush 12/30 Write block 222 len 40960 data: 11 7 9 7 6 7 8 9 12 17 13/30 Write block 540 len 16384 data: 11 11 9 8 14/30 Write block 396 len 4096 data: 14 15/30 Read block 319 len 24576 16/30 Read block 470 len 32768 17/30 Read block 148 len 16384 18/30 Write block 371 len 28672 data: 10 12 9 7 11 9 11 19/30 Read block 440 len 24576 20/30 Read block 314 len 24576 21/30 Write block 312 len 20480 data: 15 15 15 15 15 22/30 Read block 453 len 28672 23/30 Flush 24/30 Write block 28 len 16384 data: 9 11 13 12 25/30 Write block 436 len 16384 data: 11 14 19 20 26/30 Read block 570 len 24576 27/30 Write block 141 len 4096 data: 12 28/30 Read block 184 len 36864 29/30 Read block 336 len 8192 30/30 Read block 485 len 36864 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:15.606137211Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1832} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 278 278 278 F F F 1 020-039 158 158 158 277 277 277 T T T 2 040-059 156 156 156 275 275 275 F F F 3 060-079 156 156 156 275 275 275 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 156 156 156 275 275 275 F F F 6 120-139 159 159 159 278 278 278 F F F 7 140-159 160 160 160 279 279 279 T T T 8 160-179 160 160 160 279 279 279 F F F 9 180-199 159 159 159 278 278 278 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 160 160 160 280 280 280 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 160 160 160 279 279 279 T T T 22 440-459 160 160 160 279 279 279 F F F 23 460-479 159 159 159 278 278 278 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 160 160 160 279 279 279 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 160, Max flush: 280 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 278 278 278 F F F 1 020-039 161 161 161 281 281 281 F F F 2 040-059 156 156 156 275 275 275 F F F 3 060-079 156 156 156 275 275 275 F F F 4 080-099 148 148 148 261 261 261 F F F 5 100-119 156 156 156 275 275 275 F F F 6 120-139 159 159 159 278 278 278 F F F 7 140-159 161 161 161 281 281 281 F F F 8 160-179 160 160 160 279 279 279 F F F 9 180-199 159 159 159 278 278 278 F F F 10 200-219 150 150 150 263 263 263 F F F 11 220-239 160 160 160 280 280 280 F F F 12 240-259 152 152 152 268 268 268 F F F 13 260-279 152 152 152 269 269 269 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 161 161 161 281 281 281 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 159 159 159 278 278 278 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 160 160 160 279 279 279 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 161, Max flush: 281 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.867363881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:19.910274497Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.910654067Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.941450325Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.942028275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.942062891Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.94207757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.942098188Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} {"msg":"Crucible 00b5d64e-8a2d-4bdf-a7ba-5ad3d00a74c0 has session id: 44577f55-38e0-4ee0-99dd-8bb75f2aa288","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.942714763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Upstairs opts: Upstairs UUID: 00b5d64e-8a2d-4bdf-a7ba-5ad3d00a74c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.94275323Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Crucible stats registered with UUID: 00b5d64e-8a2d-4bdf-a7ba-5ad3d00a74c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.942770868Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:19.942783976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943270106Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943291503Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} {"msg":"00b5d64e-8a2d-4bdf-a7ba-5ad3d00a74c0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943360296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943387553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"0","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.94341755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"0","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943435588Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"1","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943461626Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"1","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943487613Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"2","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.94351659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"2","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943547967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943576264Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.94360902Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"ds_connection connected from Ok(127.0.0.1:46992)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943634538Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"ds_connection connected from Ok(127.0.0.1:54354)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943660625Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"ds_connection connected from Ok(127.0.0.1:60546)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.943789402Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.944325016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"2","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.944495818Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"1","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.944602957Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"0","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.944982967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"2","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.945112704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"1","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.945446789Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"0","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.94562343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.945747737Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[0]R flush_numbers[0..12]: [278, 281, 275, 275, 261, 275, 278, 281, 279, 278, 263, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946070773Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[0]R generation[0..12]: [159, 161, 156, 156, 148, 156, 159, 161, 160, 159, 150, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946094781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946110999Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[1]R flush_numbers[0..12]: [278, 281, 275, 275, 261, 275, 278, 281, 279, 278, 263, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946125268Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[1]R generation[0..12]: [159, 161, 156, 156, 148, 156, 159, 161, 160, 159, 150, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946139026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946152815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[2]R flush_numbers[0..12]: [278, 281, 275, 275, 261, 275, 278, 281, 279, 278, 263, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946167723Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[2]R generation[0..12]: [159, 161, 156, 156, 148, 156, 159, 161, 160, 159, 150, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946181932Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946195871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946208759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946221758Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Next flush: 282","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-03-18T05:07:19.946240476Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946253465Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"":"downstairs","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946266883Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946278952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"0","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.94629429Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"1","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946307419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"client":"2","":"downstairs","session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946320367Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946332646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"00b5d64e-8a2d-4bdf-a7ba-5ad3d00a74c0 is now active with session: 44577f55-38e0-4ee0-99dd-8bb75f2aa288","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946345225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946356984Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840,"session_id":"44577f55-38e0-4ee0-99dd-8bb75f2aa288"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946369022Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} {"msg":"Activated sub_volume 00b5d64e-8a2d-4bdf-a7ba-5ad3d00a74c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:19.946380001Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 212 len 32768 data: 12 17 16 14 12 13 15 14 02/30 Read block 13 len 32768 03/30 Write block 73 len 28672 data: 9 11 13 15 16 16 18 04/30 Read block 124 len 24576 05/30 Read block 52 len 4096 06/30 Read block 373 len 28672 07/30 Read block 275 len 36864 08/30 Read block 281 len 8192 09/30 Read block 74 len 4096 10/30 Write block 26 len 24576 data: 8 7 10 12 14 13 11/30 Read block 539 len 24576 12/30 Read block 66 len 4096 13/30 Read block 541 len 12288 14/30 Read block 543 len 16384 15/30 Write block 256 len 24576 data: 9 12 12 13 12 11 16/30 Write block 244 len 24576 data: 14 11 10 10 9 9 17/30 Read block 202 len 24576 18/30 Read block 108 len 4096 19/30 Read block 255 len 32768 20/30 Read block 465 len 40960 21/30 Read block 307 len 20480 22/30 Read block 192 len 24576 23/30 Write block 97 len 28672 data: 12 12 10 12 11 11 8 24/30 Read block 92 len 40960 25/30 Read block 481 len 32768 26/30 Flush 27/30 Write block 54 len 12288 data: 12 14 12 28/30 Write block 265 len 32768 data: 11 11 11 14 15 15 15 13 29/30 Read block 116 len 32768 30/30 Write block 107 len 36864 data: 13 12 11 11 14 14 10 9 14 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:19.968552059Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1840} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 278 278 278 F F F 1 020-039 162 162 162 282 282 282 F F F 2 040-059 156 156 156 275 275 275 T T T 3 060-079 162 162 162 282 282 282 F F F 4 080-099 162 162 162 282 282 282 F F F 5 100-119 162 162 162 282 282 282 T T T 6 120-139 159 159 159 278 278 278 F F F 7 140-159 161 161 161 281 281 281 F F F 8 160-179 160 160 160 279 279 279 F F F 9 180-199 159 159 159 278 278 278 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 160 160 160 280 280 280 F F F 12 240-259 162 162 162 282 282 282 F F F 13 260-279 162 162 162 282 282 282 T T T 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 161 161 161 281 281 281 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 159 159 159 278 278 278 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 160 160 160 279 279 279 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 162, Max flush: 282 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 278 278 278 F F F 1 020-039 162 162 162 282 282 282 F F F 2 040-059 163 163 163 283 283 283 F F F 3 060-079 162 162 162 282 282 282 F F F 4 080-099 162 162 162 282 282 282 F F F 5 100-119 163 163 163 283 283 283 F F F 6 120-139 159 159 159 278 278 278 F F F 7 140-159 161 161 161 281 281 281 F F F 8 160-179 160 160 160 279 279 279 F F F 9 180-199 159 159 159 278 278 278 F F F 10 200-219 162 162 162 282 282 282 F F F 11 220-239 160 160 160 280 280 280 F F F 12 240-259 162 162 162 282 282 282 F F F 13 260-279 163 163 163 283 283 283 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 161 161 161 281 281 281 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 159 159 159 278 278 278 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 160 160 160 279 279 279 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 163, Max flush: 283 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.220282769Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.26429132Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.264912105Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.264946721Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.26496144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.264976528Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} {"msg":"Crucible 1f8c76a3-a170-4302-9045-6dba59913deb has session id: 4f23eaba-050f-42fc-ab4d-ecd007b1dad9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.265573206Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Upstairs opts: Upstairs UUID: 1f8c76a3-a170-4302-9045-6dba59913deb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.265608392Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Crucible stats registered with UUID: 1f8c76a3-a170-4302-9045-6dba59913deb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.265623861Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:24.265636589Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266148196Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266177183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} {"msg":"1f8c76a3-a170-4302-9045-6dba59913deb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266247856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266265464Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"0","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266293811Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"0","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266313019Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"1","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266327547Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"1","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266351995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"2","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266433536Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"2","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266518777Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266559703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.26658804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50871)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266616817Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53177)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266734465Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53241)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.266765092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.267210005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"0","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.267239222Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"2","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.267909722Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"0","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.267964546Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"2","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.268593471Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.26869887Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"1","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.268721867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269280129Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"1","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[0]R flush_numbers[0..12]: [278, 282, 283, 282, 282, 283, 278, 281, 279, 278, 282, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269801345Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[0]R generation[0..12]: [159, 162, 163, 162, 162, 163, 159, 161, 160, 159, 162, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269821253Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269836111Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[1]R flush_numbers[0..12]: [278, 282, 283, 282, 282, 283, 278, 281, 279, 278, 282, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.26984992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[1]R generation[0..12]: [159, 162, 163, 162, 162, 163, 159, 161, 160, 159, 162, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269863579Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269877327Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[2]R flush_numbers[0..12]: [278, 282, 283, 282, 282, 283, 278, 281, 279, 278, 282, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269891026Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[2]R generation[0..12]: [159, 162, 163, 162, 162, 163, 159, 161, 160, 159, 162, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269904964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269919123Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269936821Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.26995043Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269963248Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.269982356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"":"downstairs","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270008683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270022382Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"0","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270035781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"1","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270048719Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"client":"2","":"downstairs","session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270061728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270073867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"1f8c76a3-a170-4302-9045-6dba59913deb is now active with session: 4f23eaba-050f-42fc-ab4d-ecd007b1dad9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270086545Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270098454Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848,"session_id":"4f23eaba-050f-42fc-ab4d-ecd007b1dad9"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270110503Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} {"msg":"Activated sub_volume 1f8c76a3-a170-4302-9045-6dba59913deb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:24.270121412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 184 len 4096 data: 16 02/30 Read block 249 len 8192 03/30 Write block 471 len 28672 data: 7 8 11 10 11 13 17 04/30 Read block 495 len 32768 05/30 Write block 374 len 32768 data: 8 12 10 12 13 14 17 20 06/30 Read block 9 len 16384 07/30 Read block 96 len 4096 08/30 Write block 202 len 20480 data: 12 9 11 11 13 09/30 Read block 338 len 16384 10/30 Read block 583 len 12288 11/30 Read block 122 len 12288 12/30 Write block 564 len 24576 data: 14 14 14 14 11 10 13/30 Write block 37 len 24576 data: 16 16 17 16 19 15 14/30 Write block 8 len 28672 data: 11 11 10 13 14 15 17 15/30 Read block 217 len 36864 16/30 Read block 332 len 20480 17/30 Read block 561 len 16384 18/30 Read block 210 len 24576 19/30 Write block 465 len 36864 data: 8 7 9 9 7 6 8 9 12 20/30 Write block 423 len 32768 data: 8 10 13 13 16 14 13 12 21/30 Read block 92 len 32768 22/30 Write block 33 len 36864 data: 14 16 15 14 17 17 18 17 20 23/30 Read block 555 len 12288 24/30 Write block 29 len 28672 data: 13 15 14 12 15 17 16 25/30 Read block 60 len 12288 26/30 Read block 465 len 36864 27/30 Write block 215 len 8192 data: 15 13 28/30 Write block 566 len 32768 data: 15 15 12 11 12 10 16 19 29/30 Write block 59 len 28672 data: 20 17 14 12 12 10 10 30/30 Write block 41 len 8192 data: 21 16 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:24.291469285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1848} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 278 278 278 T T T 1 020-039 162 162 162 282 282 282 T T T 2 040-059 163 163 163 283 283 283 T T T 3 060-079 162 162 162 282 282 282 T T T 4 080-099 162 162 162 282 282 282 F F F 5 100-119 163 163 163 283 283 283 F F F 6 120-139 159 159 159 278 278 278 F F F 7 140-159 161 161 161 281 281 281 F F F 8 160-179 160 160 160 279 279 279 F F F 9 180-199 159 159 159 278 278 278 T T T 10 200-219 162 162 162 282 282 282 T T T 11 220-239 160 160 160 280 280 280 F F F 12 240-259 162 162 162 282 282 282 F F F 13 260-279 163 163 163 283 283 283 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 160 160 160 280 280 280 T T T 19 380-399 160 160 160 280 280 280 T T T 20 400-419 156 156 156 275 275 275 F F F 21 420-439 161 161 161 281 281 281 T T T 22 440-459 160 160 160 279 279 279 F F F 23 460-479 159 159 159 278 278 278 T T T 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 160 160 160 279 279 279 T T T 29 580-599 160 160 160 279 279 279 F F F Max gen: 163, Max flush: 283 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 284 284 284 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 164 164 164 284 284 284 F F F 3 060-079 164 164 164 284 284 284 F F F 4 080-099 162 162 162 282 282 282 F F F 5 100-119 163 163 163 283 283 283 F F F 6 120-139 159 159 159 278 278 278 F F F 7 140-159 161 161 161 281 281 281 F F F 8 160-179 160 160 160 279 279 279 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 164 164 164 284 284 284 F F F 11 220-239 160 160 160 280 280 280 F F F 12 240-259 162 162 162 282 282 282 F F F 13 260-279 163 163 163 283 283 283 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 164 164 164 284 284 284 F F F 19 380-399 164 164 164 284 284 284 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 164 164 164 284 284 284 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 164 164 164 284 284 284 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 164 164 164 284 284 284 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 164, Max flush: 284 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.562830185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.606639287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.607297218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.607337964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.607360182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.60737895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} {"msg":"Crucible aaf8e5f5-3a00-4467-96fc-d6bc8183dc0e has session id: 3932d786-a628-40d6-af40-2c218aabeab9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.607969808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Upstairs opts: Upstairs UUID: aaf8e5f5-3a00-4467-96fc-d6bc8183dc0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608023952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Crucible stats registered with UUID: aaf8e5f5-3a00-4467-96fc-d6bc8183dc0e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608053329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:38.608079587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.60842704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608454957Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} {"msg":"aaf8e5f5-3a00-4467-96fc-d6bc8183dc0e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608507242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608534349Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"0","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608576595Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"0","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608607121Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"1","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608636878Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"1","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608662246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"2","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608694603Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"2","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608723009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608751206Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608777314Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59719)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608824019Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40361)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608850386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33095)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.608877623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.60938521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"1","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.609416267Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"0","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.610005876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"0","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.61015595Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"1","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.610610003Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.610725511Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.610746718Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"2","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.61130116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"2","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[0]R flush_numbers[0..12]: [284, 284, 284, 284, 282, 283, 278, 281, 279, 284, 284, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611826256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[0]R generation[0..12]: [164, 164, 164, 164, 162, 163, 159, 161, 160, 164, 164, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611851373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611866602Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[1]R flush_numbers[0..12]: [284, 284, 284, 284, 282, 283, 278, 281, 279, 284, 284, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.61188046Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[1]R generation[0..12]: [164, 164, 164, 164, 162, 163, 159, 161, 160, 164, 164, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611894239Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611908317Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[2]R flush_numbers[0..12]: [284, 284, 284, 284, 282, 283, 278, 281, 279, 284, 284, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611922426Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[2]R generation[0..12]: [164, 164, 164, 164, 162, 163, 159, 161, 160, 164, 164, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611936184Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611950063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Max found gen is 165","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.611962871Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Generation requested: 166 >= found:165","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.61197576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:07:38.611988369Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612033684Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"":"downstairs","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612048073Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612063751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"0","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.61207757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"1","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612090808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"client":"2","":"downstairs","session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612103777Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612115856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"aaf8e5f5-3a00-4467-96fc-d6bc8183dc0e is now active with session: 3932d786-a628-40d6-af40-2c218aabeab9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612128874Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612140633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856,"session_id":"3932d786-a628-40d6-af40-2c218aabeab9"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612152642Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} {"msg":"Activated sub_volume aaf8e5f5-3a00-4467-96fc-d6bc8183dc0e","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:38.612163811Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 98 len 32768 data: 13 11 13 12 12 9 6 8 02/30 Read block 49 len 12288 03/30 Read block 164 len 28672 04/30 Write block 471 len 20480 data: 9 10 13 11 12 05/30 Flush 06/30 Write block 232 len 36864 data: 17 15 14 12 12 11 10 13 11 07/30 Write block 424 len 32768 data: 11 14 14 17 15 14 13 12 08/30 Flush 09/30 Write block 236 len 16384 data: 13 12 11 14 10/30 Read block 135 len 40960 11/30 Write block 390 len 24576 data: 8 11 14 14 15 12 12/30 Read block 548 len 12288 13/30 Read block 283 len 28672 14/30 Write block 590 len 16384 data: 11 9 7 9 15/30 Read block 172 len 20480 16/30 Write block 42 len 40960 data: 17 13 13 10 10 10 9 10 13 13 17/30 Flush 18/30 Write block 580 len 20480 data: 8 9 7 8 10 19/30 Read block 82 len 40960 20/30 Read block 446 len 40960 21/30 Read block 115 len 32768 22/30 Write block 344 len 8192 data: 9 8 23/30 Write block 13 len 28672 data: 16 18 16 16 16 15 16 24/30 Read block 345 len 20480 25/30 Write block 82 len 8192 data: 17 21 26/30 Flush 27/30 Write block 364 len 8192 data: 10 11 28/30 Write block 432 len 20480 data: 13 10 12 10 12 29/30 Write block 139 len 28672 data: 12 12 13 10 10 11 10 30/30 Write block 565 len 16384 data: 15 16 16 13 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:38.633253801Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 288 288 288 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 166 166 166 287 287 287 F F F 3 060-079 164 164 164 284 284 284 F F F 4 080-099 166 166 166 288 288 288 F F F 5 100-119 166 166 166 285 285 285 F F F 6 120-139 159 159 159 278 278 278 T T T 7 140-159 161 161 161 281 281 281 T T T 8 160-179 160 160 160 279 279 279 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 164 164 164 284 284 284 F F F 11 220-239 166 166 166 287 287 287 F F F 12 240-259 166 166 166 286 286 286 F F F 13 260-279 163 163 163 283 283 283 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 164 164 164 284 284 284 T T T 19 380-399 166 166 166 287 287 287 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 166 166 166 286 286 286 T T T 22 440-459 160 160 160 279 279 279 F F F 23 460-479 166 166 166 285 285 285 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 164 164 164 284 284 284 T T T 29 580-599 166 166 166 288 288 288 F F F Max gen: 166, Max flush: 288 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 288 288 288 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 166 166 166 287 287 287 F F F 3 060-079 164 164 164 284 284 284 F F F 4 080-099 166 166 166 288 288 288 F F F 5 100-119 166 166 166 285 285 285 F F F 6 120-139 167 167 167 289 289 289 F F F 7 140-159 167 167 167 289 289 289 F F F 8 160-179 160 160 160 279 279 279 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 164 164 164 284 284 284 F F F 11 220-239 166 166 166 287 287 287 F F F 12 240-259 166 166 166 286 286 286 F F F 13 260-279 163 163 163 283 283 283 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 160 160 160 280 280 280 F F F 16 320-339 152 152 152 268 268 268 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 167 167 167 289 289 289 F F F 19 380-399 166 166 166 287 287 287 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 167 167 167 289 289 289 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 166 166 166 285 285 285 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 166 166 166 288 288 288 F F F Max gen: 167, Max flush: 289 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.873068064Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.91760121Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.918249242Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.918298787Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.918323484Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.918348882Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} {"msg":"Crucible 81e1423d-57df-47a2-b443-94611664697d has session id: dbbc1330-e320-4e33-8050-1ae047136143","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.918876897Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Upstairs opts: Upstairs UUID: 81e1423d-57df-47a2-b443-94611664697d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.918912853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Crucible stats registered with UUID: 81e1423d-57df-47a2-b443-94611664697d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.918928041Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:42.91894075Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919342328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919369135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} {"msg":"81e1423d-57df-47a2-b443-94611664697d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919430929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919453497Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"0","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919482134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"0","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919495822Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"1","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.91951872Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"1","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919545437Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"2","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919572834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"2","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919598182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919623659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919647606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"ds_connection connected from Ok(127.0.0.1:41173)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919674934Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"ds_connection connected from Ok(127.0.0.1:54427)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919855915Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"ds_connection connected from Ok(127.0.0.1:57102)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.919962224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.920444833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"2","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.920483109Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"0","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.921038201Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"1","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.921157189Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"0","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.921198664Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"2","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.921599693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"1","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.921762516Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.921846997Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[0]R flush_numbers[0..12]: [288, 284, 287, 284, 288, 285, 289, 289, 279, 284, 284, 287]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922126908Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[0]R generation[0..12]: [166, 164, 166, 164, 166, 166, 167, 167, 160, 164, 164, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922151725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922166894Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[1]R flush_numbers[0..12]: [288, 284, 287, 284, 288, 285, 289, 289, 279, 284, 284, 287]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922182552Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[1]R generation[0..12]: [166, 164, 166, 164, 166, 166, 167, 167, 160, 164, 164, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.92219648Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922210549Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[2]R flush_numbers[0..12]: [288, 284, 287, 284, 288, 285, 289, 289, 279, 284, 284, 287]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922224408Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[2]R generation[0..12]: [166, 164, 166, 164, 166, 166, 167, 167, 160, 164, 164, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922237836Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922255334Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922268383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922281362Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.92229408Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922307169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"":"downstairs","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922320478Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922333206Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"Wait for a query_work_queue command to finish before sending IO client":"0","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922355774Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"1","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922368842Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"client":"2","":"downstairs","session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922381701Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.92239371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"81e1423d-57df-47a2-b443-94611664697d is now active with session: dbbc1330-e320-4e33-8050-1ae047136143","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922406329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922418057Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864,"session_id":"dbbc1330-e320-4e33-8050-1ae047136143"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922430006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} {"msg":"Activated sub_volume 81e1423d-57df-47a2-b443-94611664697d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:42.922441005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 337 len 8192 data: 9 10 02/30 Read block 175 len 28672 03/30 Read block 544 len 16384 04/30 Read block 196 len 4096 05/30 Read block 162 len 4096 06/30 Flush 07/30 Write block 90 len 8192 data: 18 16 08/30 Read block 261 len 8192 09/30 Write block 590 len 20480 data: 12 10 8 10 8 10/30 Write block 314 len 28672 data: 16 16 16 16 13 10 12 11/30 Write block 55 len 40960 data: 15 13 14 18 21 18 15 13 13 11 12/30 Write block 5 len 4096 data: 11 13/30 Write block 75 len 8192 data: 14 16 14/30 Flush 15/30 Write block 382 len 24576 data: 21 23 20 15 14 9 16/30 Read block 320 len 40960 17/30 Write block 100 len 24576 data: 14 13 13 10 7 9 18/30 Write block 266 len 16384 data: 12 12 15 16 19/30 Flush 20/30 Write block 208 len 8192 data: 11 11 21/30 Write block 160 len 40960 data: 19 14 12 13 12 10 14 12 14 15 22/30 Read block 64 len 16384 23/30 Read block 10 len 24576 24/30 Read block 24 len 28672 25/30 Read block 59 len 12288 26/30 Read block 389 len 24576 27/30 Read block 133 len 8192 28/30 Read block 476 len 20480 29/30 Read block 0 len 36864 30/30 Read block 462 len 16384 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:42.941755651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1864} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 291 291 291 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 168 168 168 291 291 291 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 168 168 168 292 292 292 F F F 6 120-139 167 167 167 289 289 289 F F F 7 140-159 167 167 167 289 289 289 F F F 8 160-179 160 160 160 279 279 279 T T T 9 180-199 164 164 164 284 284 284 F F F 10 200-219 164 164 164 284 284 284 T T T 11 220-239 166 166 166 287 287 287 F F F 12 240-259 166 166 166 286 286 286 F F F 13 260-279 168 168 168 292 292 292 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 168 168 168 291 291 291 F F F 16 320-339 168 168 168 291 291 291 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 167 167 167 289 289 289 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 167 167 167 289 289 289 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 166 166 166 285 285 285 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 168 168 168 291 291 291 F F F Max gen: 168, Max flush: 292 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 291 291 291 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 168 168 168 291 291 291 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 168 168 168 292 292 292 F F F 6 120-139 167 167 167 289 289 289 F F F 7 140-159 167 167 167 289 289 289 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 169 169 169 293 293 293 F F F 11 220-239 166 166 166 287 287 287 F F F 12 240-259 166 166 166 286 286 286 F F F 13 260-279 168 168 168 292 292 292 F F F 14 280-299 158 158 158 276 276 276 F F F 15 300-319 168 168 168 291 291 291 F F F 16 320-339 168 168 168 291 291 291 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 167 167 167 289 289 289 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 167 167 167 289 289 289 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 166 166 166 285 285 285 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 168 168 168 291 291 291 F F F Max gen: 169, Max flush: 293 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.160247898Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:47.205423917Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.205832185Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.23704214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.23761653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.237650197Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.237668415Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.237683083Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} {"msg":"Crucible ccb4156f-e9e0-4a44-bd04-1b179addb67d has session id: 68cc062f-c449-4359-a16a-3bb4dc92b17f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.23829281Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Upstairs opts: Upstairs UUID: ccb4156f-e9e0-4a44-bd04-1b179addb67d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.238335455Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Crucible stats registered with UUID: ccb4156f-e9e0-4a44-bd04-1b179addb67d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.238353673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:47.238367052Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.238888847Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.238947991Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} {"msg":"ccb4156f-e9e0-4a44-bd04-1b179addb67d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239003865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239030693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"0","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239086107Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"0","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239102775Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"1","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239117324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"1","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239130502Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"2","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.23914553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"2","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239167808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239194865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239218273Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63944)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239265438Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39789)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239466097Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59213)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.239497624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.240076053Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"1","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.240117379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"2","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.240296621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"0","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.240833794Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"1","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.2408746Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"2","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241005727Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"0","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241439841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241507924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[0]R flush_numbers[0..12]: [291, 284, 291, 291, 291, 292, 289, 289, 293, 284, 293, 287]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241613733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[0]R generation[0..12]: [168, 164, 168, 168, 168, 168, 167, 167, 169, 164, 169, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.24163929Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241654749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[1]R flush_numbers[0..12]: [291, 284, 291, 291, 291, 292, 289, 289, 293, 284, 293, 287]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241668798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[1]R generation[0..12]: [168, 164, 168, 168, 168, 168, 167, 167, 169, 164, 169, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241682566Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241696975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[2]R flush_numbers[0..12]: [291, 284, 291, 291, 291, 292, 289, 289, 293, 284, 293, 287]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241710713Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[2]R generation[0..12]: [168, 164, 168, 168, 168, 168, 167, 167, 169, 164, 169, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241724462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.24173827Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241751189Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241764297Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241777126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241796214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"":"downstairs","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241809563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241821511Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"0","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.24183622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"1","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241849339Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"client":"2","":"downstairs","session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241862297Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241874246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"ccb4156f-e9e0-4a44-bd04-1b179addb67d is now active with session: 68cc062f-c449-4359-a16a-3bb4dc92b17f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241886975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241898714Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872,"session_id":"68cc062f-c449-4359-a16a-3bb4dc92b17f"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241910732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} {"msg":"Activated sub_volume ccb4156f-e9e0-4a44-bd04-1b179addb67d","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:47.241921671Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 152 len 4096 data: 10 02/30 Read block 15 len 40960 03/30 Write block 295 len 4096 data: 18 04/30 Read block 325 len 24576 05/30 Write block 264 len 24576 data: 9 12 13 13 16 17 06/30 Read block 160 len 4096 07/30 Write block 9 len 24576 data: 12 11 14 15 17 19 08/30 Write block 108 len 36864 data: 13 12 12 15 15 11 10 15 13 09/30 Read block 46 len 16384 10/30 Read block 17 len 12288 11/30 Read block 106 len 16384 12/30 Read block 192 len 24576 13/30 Read block 213 len 36864 14/30 Write block 8 len 40960 data: 12 13 12 15 16 18 20 17 17 17 15/30 Read block 527 len 32768 16/30 Write block 145 len 16384 data: 11 11 12 12 17/30 Flush 18/30 Write block 5 len 16384 data: 12 14 14 13 19/30 Read block 149 len 40960 20/30 Read block 553 len 32768 21/30 Read block 271 len 28672 22/30 Write block 225 len 36864 data: 8 7 8 9 10 13 18 18 16 23/30 Read block 265 len 36864 24/30 Read block 186 len 36864 25/30 Write block 245 len 24576 data: 12 11 11 10 10 7 26/30 Read block 418 len 40960 27/30 Write block 436 len 12288 data: 13 15 20 28/30 Write block 366 len 4096 data: 11 29/30 Read block 224 len 20480 30/30 Read block 161 len 28672 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:47.264110777Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1872} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 294 294 294 T T T 1 020-039 164 164 164 284 284 284 F F F 2 040-059 168 168 168 291 291 291 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 170 170 170 294 294 294 F F F 6 120-139 167 167 167 289 289 289 F F F 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 169 169 169 293 293 293 F F F 11 220-239 166 166 166 287 287 287 T T T 12 240-259 166 166 166 286 286 286 T T T 13 260-279 170 170 170 294 294 294 F F F 14 280-299 170 170 170 294 294 294 F F F 15 300-319 168 168 168 291 291 291 F F F 16 320-339 168 168 168 291 291 291 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 167 167 167 289 289 289 T T T 19 380-399 168 168 168 292 292 292 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 167 167 167 289 289 289 T T T 22 440-459 160 160 160 279 279 279 F F F 23 460-479 166 166 166 285 285 285 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 168 168 168 291 291 291 F F F Max gen: 170, Max flush: 294 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 295 295 295 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 168 168 168 291 291 291 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 170 170 170 294 294 294 F F F 6 120-139 167 167 167 289 289 289 F F F 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 169 169 169 293 293 293 F F F 11 220-239 171 171 171 295 295 295 F F F 12 240-259 171 171 171 295 295 295 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 170 170 170 294 294 294 F F F 15 300-319 168 168 168 291 291 291 F F F 16 320-339 168 168 168 291 291 291 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 171 171 171 295 295 295 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 171 171 171 295 295 295 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 166 166 166 285 285 285 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 F F F 26 520-539 156 156 156 275 275 275 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 167 167 167 289 289 289 F F F 29 580-599 168 168 168 291 291 291 F F F Max gen: 171, Max flush: 295 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.533220216Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.577915984Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.578588874Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.578623961Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.578638379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.578652567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} {"msg":"Crucible ca82bd6d-f296-493b-be3a-64b75d48084b has session id: c4384afe-c8a0-4357-b6b6-2c2003c19940","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579240036Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Upstairs opts: Upstairs UUID: ca82bd6d-f296-493b-be3a-64b75d48084b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579288421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Crucible stats registered with UUID: ca82bd6d-f296-493b-be3a-64b75d48084b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579313749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:51.579335296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579806677Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579839814Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} {"msg":"ca82bd6d-f296-493b-be3a-64b75d48084b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579904657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579934804Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"0","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.579983379Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"0","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580014016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"1","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580043353Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"1","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.58006833Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"2","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580094797Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"2","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580118575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580141752Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.58016468Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"ds_connection connected from Ok(127.0.0.1:46036)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580189387Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"ds_connection connected from Ok(127.0.0.1:58861)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580214735Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"ds_connection connected from Ok(127.0.0.1:51764)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580240002Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.580973516Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"2","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.581162486Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"1","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.581199032Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"0","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.581672563Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"2","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.58179615Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"1","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.581870642Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"0","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582352782Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.58246672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[0]R flush_numbers[0..12]: [295, 284, 291, 291, 291, 294, 289, 294, 293, 284, 293, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582535163Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[0]R generation[0..12]: [171, 164, 168, 168, 168, 170, 167, 170, 169, 164, 169, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582564779Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582581948Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[1]R flush_numbers[0..12]: [295, 284, 291, 291, 291, 294, 289, 294, 293, 284, 293, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582597016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[1]R generation[0..12]: [171, 164, 168, 168, 168, 170, 167, 170, 169, 164, 169, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582611745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582626233Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[2]R flush_numbers[0..12]: [295, 284, 291, 291, 291, 294, 289, 294, 293, 284, 293, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582640712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[2]R generation[0..12]: [171, 164, 168, 168, 168, 170, 167, 170, 169, 164, 169, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.58265466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582677748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582699896Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582719594Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Next flush: 296","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:07:51.582735152Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.58275525Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"":"downstairs","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582768568Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582780647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"0","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582793756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"1","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582806944Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"client":"2","":"downstairs","session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582820053Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582832322Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"ca82bd6d-f296-493b-be3a-64b75d48084b is now active with session: c4384afe-c8a0-4357-b6b6-2c2003c19940","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.5828452Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582857329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880,"session_id":"c4384afe-c8a0-4357-b6b6-2c2003c19940"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582869548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} {"msg":"Activated sub_volume ca82bd6d-f296-493b-be3a-64b75d48084b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:51.582881877Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 566 len 4096 data: 17 02/30 Read block 0 len 32768 03/30 Read block 94 len 40960 04/30 Write block 251 len 36864 data: 8 9 9 11 11 10 13 13 14 05/30 Flush 06/30 Write block 132 len 4096 data: 8 07/30 Flush 08/30 Write block 531 len 8192 data: 7 6 09/30 Write block 137 len 4096 data: 12 10/30 Write block 206 len 36864 data: 14 11 12 12 9 13 13 18 17 11/30 Flush 12/30 Write block 504 len 16384 data: 14 13 13 11 13/30 Read block 317 len 24576 14/30 Read block 468 len 4096 15/30 Read block 448 len 24576 16/30 Read block 118 len 8192 17/30 Write block 299 len 40960 data: 18 15 15 18 18 19 17 14 13 13 18/30 Read block 55 len 24576 19/30 Write block 374 len 16384 data: 9 13 11 13 20/30 Read block 153 len 12288 21/30 Read block 383 len 12288 22/30 Read block 316 len 36864 23/30 Write block 567 len 36864 data: 17 14 12 13 11 17 20 20 20 24/30 Write block 477 len 8192 data: 18 16 25/30 Read block 437 len 4096 26/30 Write block 509 len 8192 data: 10 14 27/30 Write block 581 len 40960 data: 10 8 9 11 12 11 12 10 12 13 28/30 Read block 67 len 32768 29/30 Read block 446 len 40960 30/30 Write block 238 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:51.603031015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1880} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 295 295 295 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 168 168 168 291 291 291 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 170 170 170 294 294 294 F F F 6 120-139 172 172 172 298 298 298 F F F 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 172 172 172 298 298 298 F F F 11 220-239 171 171 171 295 295 295 T T T 12 240-259 172 172 172 296 296 296 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 170 170 170 294 294 294 T T T 15 300-319 168 168 168 291 291 291 T T T 16 320-339 168 168 168 291 291 291 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 171 171 171 295 295 295 T T T 19 380-399 168 168 168 292 292 292 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 171 171 171 295 295 295 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 166 166 166 285 285 285 T T T 24 480-499 159 159 159 278 278 278 F F F 25 500-519 158 158 158 277 277 277 T T T 26 520-539 172 172 172 298 298 298 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 172 172 172 296 296 296 T T T 29 580-599 168 168 168 291 291 291 T T T Max gen: 172, Max flush: 298 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 295 295 295 F F F 1 020-039 164 164 164 284 284 284 F F F 2 040-059 168 168 168 291 291 291 F F F 3 060-079 168 168 168 291 291 291 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 170 170 170 294 294 294 F F F 6 120-139 172 172 172 298 298 298 F F F 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 172 172 172 298 298 298 F F F 11 220-239 173 173 173 299 299 299 F F F 12 240-259 172 172 172 296 296 296 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 173 173 173 299 299 299 F F F 16 320-339 168 168 168 291 291 291 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 171 171 171 295 295 295 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 173 173 173 299 299 299 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 173 173 173 299 299 299 F F F 26 520-539 172 172 172 298 298 298 F F F 27 540-559 160 160 160 280 280 280 F F F 28 560-579 173 173 173 299 299 299 F F F 29 580-599 173 173 173 299 299 299 F F F Max gen: 173, Max flush: 299 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.85941921Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.90352808Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.904134287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.904179113Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.90420338Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.904222398Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} {"msg":"Crucible e4d6b0ee-7529-4e97-85a7-565834c18540 has session id: 39029df7-8c11-4102-bc6b-84af995b5fe2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.904882019Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Upstairs opts: Upstairs UUID: e4d6b0ee-7529-4e97-85a7-565834c18540, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.904925015Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Crucible stats registered with UUID: e4d6b0ee-7529-4e97-85a7-565834c18540","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.904943763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:55.904957561Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905507164Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905531712Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} {"msg":"e4d6b0ee-7529-4e97-85a7-565834c18540 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905600294Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905618562Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"0","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905647149Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"0","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905661748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"1","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905675856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"1","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905688905Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"2","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905703163Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"2","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905719552Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.90573354Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905747169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48124)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905767057Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51811)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905790614Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"ds_connection connected from Ok(127.0.0.1:57683)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.905815032Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.906514659Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"0","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.906663853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"2","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.907004288Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"1","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.907086459Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"0","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.907300377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"2","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.907659619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"1","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.907771328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.907798135Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[0]R flush_numbers[0..12]: [295, 284, 291, 291, 291, 294, 298, 294, 293, 284, 298, 299]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908221301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[0]R generation[0..12]: [171, 164, 168, 168, 168, 170, 172, 170, 169, 164, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908242169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908257627Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[1]R flush_numbers[0..12]: [295, 284, 291, 291, 291, 294, 298, 294, 293, 284, 298, 299]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908271866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[1]R generation[0..12]: [171, 164, 168, 168, 168, 170, 172, 170, 169, 164, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908285884Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908300063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[2]R flush_numbers[0..12]: [295, 284, 291, 291, 291, 294, 298, 294, 293, 284, 298, 299]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908313571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[2]R generation[0..12]: [171, 164, 168, 168, 168, 170, 172, 170, 169, 164, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.90832713Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908341288Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908354567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908367996Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908380964Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908404222Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"":"downstairs","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908417571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908432709Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"0","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908446117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"1","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908458996Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"client":"2","":"downstairs","session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908471835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908483873Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"e4d6b0ee-7529-4e97-85a7-565834c18540 is now active with session: 39029df7-8c11-4102-bc6b-84af995b5fe2","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908496382Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908508161Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888,"session_id":"39029df7-8c11-4102-bc6b-84af995b5fe2"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.90852021Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} {"msg":"Activated sub_volume e4d6b0ee-7529-4e97-85a7-565834c18540","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:07:55.908531169Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 593 len 12288 data: 11 9 6 03/30 Write block 21 len 20480 data: 8 10 9 9 7 04/30 Read block 6 len 8192 05/30 Write block 320 len 4096 data: 13 06/30 Read block 68 len 16384 07/30 Read block 314 len 16384 08/30 Read block 350 len 32768 09/30 Write block 65 len 24576 data: 11 8 7 7 6 6 10/30 Write block 57 len 24576 data: 15 19 22 19 16 14 11/30 Write block 334 len 16384 data: 8 6 7 10 12/30 Write block 433 len 16384 data: 11 13 11 14 13/30 Write block 533 len 16384 data: 6 8 10 12 14/30 Read block 160 len 16384 15/30 Write block 59 len 20480 data: 23 20 17 15 14 16/30 Read block 1 len 12288 17/30 Read block 319 len 40960 18/30 Write block 0 len 40960 data: 6 8 10 9 11 13 15 15 14 14 19/30 Read block 209 len 28672 20/30 Read block 220 len 12288 21/30 Read block 519 len 16384 22/30 Write block 578 len 40960 data: 15 14 9 11 9 10 12 13 12 13 23/30 Read block 248 len 20480 24/30 Write block 513 len 24576 data: 10 12 10 10 10 9 25/30 Read block 101 len 16384 26/30 Read block 419 len 24576 27/30 Write block 13 len 24576 data: 19 21 18 18 18 16 28/30 Write block 555 len 40960 data: 13 11 13 13 14 13 13 13 14 15 29/30 Write block 413 len 24576 data: 15 15 14 11 13 14 30/30 Write block 464 len 40960 data: 11 9 8 10 10 8 7 10 11 14 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:07:55.928858289Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1888} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 295 295 295 T T T 1 020-039 164 164 164 284 284 284 T T T 2 040-059 168 168 168 291 291 291 T T T 3 060-079 168 168 168 291 291 291 T T T 4 080-099 168 168 168 291 291 291 F F F 5 100-119 170 170 170 294 294 294 F F F 6 120-139 172 172 172 298 298 298 F F F 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 172 172 172 298 298 298 F F F 11 220-239 173 173 173 299 299 299 F F F 12 240-259 172 172 172 296 296 296 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 173 173 173 299 299 299 F F F 16 320-339 168 168 168 291 291 291 T T T 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 156 156 156 275 275 275 T T T 21 420-439 171 171 171 295 295 295 T T T 22 440-459 160 160 160 279 279 279 F F F 23 460-479 173 173 173 299 299 299 T T T 24 480-499 159 159 159 278 278 278 F F F 25 500-519 173 173 173 299 299 299 T T T 26 520-539 172 172 172 298 298 298 T T T 27 540-559 160 160 160 280 280 280 T T T 28 560-579 173 173 173 299 299 299 T T T 29 580-599 173 173 173 299 299 299 T T T Max gen: 173, Max flush: 299 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 300 300 300 F F F 1 020-039 174 174 174 300 300 300 F F F 2 040-059 174 174 174 300 300 300 F F F 3 060-079 174 174 174 300 300 300 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 170 170 170 294 294 294 F F F 6 120-139 172 172 172 298 298 298 F F F 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 172 172 172 298 298 298 F F F 11 220-239 173 173 173 299 299 299 F F F 12 240-259 172 172 172 296 296 296 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 173 173 173 299 299 299 F F F 16 320-339 174 174 174 300 300 300 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 174 174 174 300 300 300 F F F 21 420-439 174 174 174 300 300 300 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 174 174 174 300 300 300 F F F 24 480-499 159 159 159 278 278 278 F F F 25 500-519 174 174 174 300 300 300 F F F 26 520-539 174 174 174 300 300 300 F F F 27 540-559 174 174 174 300 300 300 F F F 28 560-579 174 174 174 300 300 300 F F F 29 580-599 174 174 174 300 300 300 F F F Max gen: 174, Max flush: 300 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.207409442Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.251017845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.251600224Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.25163441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.251649059Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.251663358Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} {"msg":"Crucible 49966052-f766-4f4f-b0e3-20bc8ecc8ed6 has session id: 38d85779-db6d-41a9-87ca-d0822e3fd103","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.252241697Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Upstairs opts: Upstairs UUID: 49966052-f766-4f4f-b0e3-20bc8ecc8ed6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.252286742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Crucible stats registered with UUID: 49966052-f766-4f4f-b0e3-20bc8ecc8ed6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.252319179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:00.252341047Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.252778031Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.252804728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} {"msg":"49966052-f766-4f4f-b0e3-20bc8ecc8ed6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.25288723Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.252910097Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"0","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.252955323Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"0","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253023386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"1","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253057742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"1","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253084529Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"2","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253111376Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"2","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253135714Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253158962Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253184439Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"ds_connection connected from Ok(127.0.0.1:36349)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253212366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"ds_connection connected from Ok(127.0.0.1:45623)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253246093Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"ds_connection connected from Ok(127.0.0.1:45202)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253276709Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.25384882Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"0","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.253880606Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"1","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.254512261Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"0","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.254666564Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"1","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.25509249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.255222596Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.255802406Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"2","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256307333Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"2","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[0]R flush_numbers[0..12]: [300, 300, 300, 300, 291, 294, 298, 294, 293, 284, 298, 299]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.25682085Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[0]R generation[0..12]: [174, 174, 174, 174, 168, 170, 172, 170, 169, 164, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256840828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256856296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[1]R flush_numbers[0..12]: [300, 300, 300, 300, 291, 294, 298, 294, 293, 284, 298, 299]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256870175Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[1]R generation[0..12]: [174, 174, 174, 174, 168, 170, 172, 170, 169, 164, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256887373Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256901751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[2]R flush_numbers[0..12]: [300, 300, 300, 300, 291, 294, 298, 294, 293, 284, 298, 299]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.25691555Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[2]R generation[0..12]: [174, 174, 174, 174, 168, 170, 172, 170, 169, 164, 172, 173]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256929109Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256942877Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","Wait for a query_work_queue command to finish before sending IO pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Max found gen is 175","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256962025Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Generation requested: 176 >= found:175","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256975264Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.256988162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257001131Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"":"downstairs","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257013989Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257028318Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"0","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257044676Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"1","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257060835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"client":"2","":"downstairs","session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T05:08:00.257076983Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.25710086Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"49966052-f766-4f4f-b0e3-20bc8ecc8ed6 is now active with session: 38d85779-db6d-41a9-87ca-d0822e3fd103","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257117119Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257131877Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896,"session_id":"38d85779-db6d-41a9-87ca-d0822e3fd103"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257147006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} {"msg":"Activated sub_volume 49966052-f766-4f4f-b0e3-20bc8ecc8ed6","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:00.257160774Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 499 len 16384 data: 12 12 13 17 03/30 Write block 525 len 28672 data: 14 12 10 11 8 8 8 04/30 Flush 05/30 Write block 115 len 36864 data: 16 14 12 13 17 15 11 11 10 06/30 Write block 423 len 28672 data: 9 12 15 15 18 16 15 07/30 Read block 448 len 40960 08/30 Flush 09/30 Write block 306 len 24576 data: 15 14 14 14 17 16 10/30 Write block 545 len 24576 data: 13 14 13 11 9 9 11/30 Write block 568 len 40960 data: 15 13 14 12 18 21 21 21 14 13 12/30 Flush 13/30 Flush 14/30 Write block 254 len 4096 data: 12 15/30 Read block 439 len 40960 16/30 Read block 29 len 36864 17/30 Read block 250 len 36864 18/30 Flush 19/30 Write block 201 len 24576 data: 14 13 10 12 12 15 20/30 Read block 186 len 32768 21/30 Write block 241 len 8192 data: 15 16 22/30 Read block 515 len 16384 23/30 Write block 137 len 4096 data: 13 24/30 Read block 451 len 24576 25/30 Read block 186 len 16384 26/30 Write block 582 len 4096 data: 10 27/30 Read block 25 len 16384 28/30 Write block 229 len 16384 data: 11 14 19 19 29/30 Write block 209 len 12288 data: 13 10 14 30/30 Read block 429 len 36864 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:00.276840292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1896} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 300 300 300 F F F 1 020-039 174 174 174 300 300 300 F F F 2 040-059 174 174 174 300 300 300 F F F 3 060-079 174 174 174 300 300 300 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 176 176 176 303 303 303 F F F 6 120-139 176 176 176 303 303 303 T T T 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 172 172 172 298 298 298 T T T 11 220-239 173 173 173 299 299 299 T T T 12 240-259 176 176 176 306 306 306 T T T 13 260-279 170 170 170 294 294 294 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 174 174 174 300 300 300 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 174 174 174 300 300 300 F F F 21 420-439 176 176 176 303 303 303 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 174 174 174 300 300 300 F F F 24 480-499 176 176 176 302 302 302 F F F 25 500-519 176 176 176 302 302 302 F F F 26 520-539 176 176 176 302 302 302 F F F 27 540-559 176 176 176 304 304 304 F F F 28 560-579 176 176 176 304 304 304 F F F 29 580-599 174 174 174 300 300 300 T T T Max gen: 176, Max flush: 306 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 300 300 300 F F F 1 020-039 174 174 174 300 300 300 F F F 2 040-059 174 174 174 300 300 300 F F F 3 060-079 174 174 174 300 300 300 F F F 4 080-099 168 168 168 291 291 291 F F F 5 100-119 176 176 176 303 303 303 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 170 170 170 294 294 294 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 177 177 177 307 307 307 F F F 11 220-239 177 177 177 307 307 307 F F F 12 240-259 177 177 177 307 307 307 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 174 174 174 300 300 300 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 168 168 168 292 292 292 F F F 20 400-419 174 174 174 300 300 300 F F F 21 420-439 176 176 176 303 303 303 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 174 174 174 300 300 300 F F F 24 480-499 176 176 176 302 302 302 F F F 25 500-519 176 176 176 302 302 302 F F F 26 520-539 176 176 176 302 302 302 F F F 27 540-559 176 176 176 304 304 304 F F F 28 560-579 176 176 176 304 304 304 F F F 29 580-599 177 177 177 307 307 307 F F F Max gen: 177, Max flush: 307 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.51508845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.560265329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.560952937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.560989673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.561011731Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.561036299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} {"msg":"Crucible 16d52dbf-3167-4923-abd1-b1b52becae46 has session id: 3ec96673-3b3f-4582-be73-0b2855686c82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.561648575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Upstairs opts: Upstairs UUID: 16d52dbf-3167-4923-abd1-b1b52becae46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.561696959Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Crucible stats registered with UUID: 16d52dbf-3167-4923-abd1-b1b52becae46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.561721627Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:04.561736075Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562179749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562201627Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} {"msg":"16d52dbf-3167-4923-abd1-b1b52becae46 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562274649Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562292337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"0","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562327114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"0","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562353221Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"1","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562384478Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"1","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562413915Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"2","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562438942Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"2","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.56246036Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562486807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562515204Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"ds_connection connected from Ok(127.0.0.1:62270)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562542371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"ds_connection connected from Ok(127.0.0.1:64847)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562569759Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"ds_connection connected from Ok(127.0.0.1:51136)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.562615834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.563237909Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"2","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.563274865Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"0","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.563304862Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"1","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.563866653Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"1","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564047605Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"0","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564193069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"2","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564514236Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564598597Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[0]R flush_numbers[0..12]: [300, 300, 300, 300, 291, 303, 307, 294, 293, 284, 307, 307]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564710355Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[0]R generation[0..12]: [174, 174, 174, 174, 168, 176, 177, 170, 169, 164, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564729473Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564744142Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[1]R flush_numbers[0..12]: [300, 300, 300, 300, 291, 303, 307, 294, 293, 284, 307, 307]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.5647582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[1]R generation[0..12]: [174, 174, 174, 174, 168, 176, 177, 170, 169, 164, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564772329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564786437Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[2]R flush_numbers[0..12]: [300, 300, 300, 300, 291, 303, 307, 294, 293, 284, 307, 307]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564799966Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[2]R generation[0..12]: [174, 174, 174, 174, 168, 176, 177, 170, 169, 164, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564813275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564826963Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564839602Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.56485275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564868689Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:04.564881787Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"":"downstairs","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564901216Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564913054Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"0","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564926473Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"1","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564939881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"client":"2","":"downstairs","session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.56495327Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564965539Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"16d52dbf-3167-4923-abd1-b1b52becae46 is now active with session: 3ec96673-3b3f-4582-be73-0b2855686c82","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564978127Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.564989946Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904,"session_id":"3ec96673-3b3f-4582-be73-0b2855686c82"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.565002035Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} {"msg":"Activated sub_volume 16d52dbf-3167-4923-abd1-b1b52becae46","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:04.565013114Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 396 len 20480 data: 15 13 18 18 16 02/30 Read block 67 len 40960 03/30 Read block 501 len 32768 04/30 Read block 340 len 40960 05/30 Write block 583 len 20480 data: 11 13 14 13 14 06/30 Read block 109 len 12288 07/30 Read block 11 len 40960 08/30 Write block 42 len 8192 data: 18 14 09/30 Read block 573 len 20480 10/30 Write block 338 len 4096 data: 11 11/30 Read block 375 len 24576 12/30 Read block 56 len 12288 13/30 Read block 556 len 8192 14/30 Read block 316 len 32768 15/30 Write block 498 len 16384 data: 12 13 13 14 16/30 Write block 542 len 12288 data: 10 9 9 17/30 Write block 86 len 20480 data: 16 17 20 18 19 18/30 Read block 436 len 36864 19/30 Read block 449 len 24576 20/30 Write block 561 len 16384 data: 14 14 15 16 21/30 Read block 533 len 24576 22/30 Write block 147 len 20480 data: 13 13 11 9 5 23/30 Read block 122 len 16384 24/30 Read block 185 len 36864 25/30 Write block 502 len 16384 data: 18 15 15 14 26/30 Read block 56 len 28672 27/30 Write block 91 len 12288 data: 17 18 15 28/30 Read block 89 len 36864 29/30 Read block 29 len 16384 30/30 Read block 277 len 40960 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 29 30 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:04.586516931Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1904} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 300 300 300 F F F 1 020-039 174 174 174 300 300 300 F F F 2 040-059 174 174 174 300 300 300 T T T 3 060-079 174 174 174 300 300 300 F F F 4 080-099 168 168 168 291 291 291 T T T 5 100-119 176 176 176 303 303 303 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 170 170 170 294 294 294 T T T 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 177 177 177 307 307 307 F F F 11 220-239 177 177 177 307 307 307 F F F 12 240-259 177 177 177 307 307 307 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 174 174 174 300 300 300 T T T 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 168 168 168 292 292 292 T T T 20 400-419 174 174 174 300 300 300 T T T 21 420-439 176 176 176 303 303 303 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 174 174 174 300 300 300 F F F 24 480-499 176 176 176 302 302 302 T T T 25 500-519 176 176 176 302 302 302 T T T 26 520-539 176 176 176 302 302 302 F F F 27 540-559 176 176 176 304 304 304 T T T 28 560-579 176 176 176 304 304 304 T T T 29 580-599 177 177 177 307 307 307 T T T Max gen: 177, Max flush: 307 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 300 300 300 F F F 1 020-039 174 174 174 300 300 300 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 174 174 174 300 300 300 F F F 4 080-099 178 178 178 308 308 308 F F F 5 100-119 176 176 176 303 303 303 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 177 177 177 307 307 307 F F F 11 220-239 177 177 177 307 307 307 F F F 12 240-259 177 177 177 307 307 307 F F F 13 260-279 170 170 170 294 294 294 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 178 178 178 308 308 308 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 178 178 178 308 308 308 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 176 176 176 303 303 303 F F F 22 440-459 160 160 160 279 279 279 F F F 23 460-479 174 174 174 300 300 300 F F F 24 480-499 178 178 178 308 308 308 F F F 25 500-519 178 178 178 308 308 308 F F F 26 520-539 176 176 176 302 302 302 F F F 27 540-559 178 178 178 308 308 308 F F F 28 560-579 178 178 178 308 308 308 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 178, Max flush: 308 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.857416029Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.901963363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.902559681Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.902594318Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.902608976Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.902624124Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} {"msg":"Crucible 382fd71d-5d8c-49a6-b198-5275b611fb8c has session id: 30a9e6b5-5418-4014-a152-36f35b54268b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.90314324Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Upstairs opts: Upstairs UUID: 382fd71d-5d8c-49a6-b198-5275b611fb8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903186556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Crucible stats registered with UUID: 382fd71d-5d8c-49a6-b198-5275b611fb8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903214263Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:18.903236951Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903694673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"task":"crutest"} {"msg":"382fd71d-5d8c-49a6-b198-5275b611fb8c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903790873Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.90382159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903856616Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"0","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903898582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"0","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.90391679Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"1","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903939887Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"1","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903964955Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"2","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.903990802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"2","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904017779Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904045206Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904075133Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52675)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904134357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63175)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.90420246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35060)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904236586Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904765161Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"1","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904806977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"0","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.904838193Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"2","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.905452009Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"0","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.905482936Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"1","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.905505624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"2","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.905978905Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906127099Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[0]R flush_numbers[0..12]: [300, 300, 308, 300, 308, 303, 307, 308, 293, 284, 307, 307]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906189483Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[0]R generation[0..12]: [174, 174, 178, 174, 178, 176, 177, 178, 169, 164, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.90621733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906237368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[1]R flush_numbers[0..12]: [300, 300, 308, 300, 308, 303, 307, 308, 293, 284, 307, 307]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906252506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[1]R generation[0..12]: [174, 174, 178, 174, 178, 176, 177, 178, 169, 164, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906266275Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906280363Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[2]R flush_numbers[0..12]: [300, 300, 308, 300, 308, 303, 307, 308, 293, 284, 307, 307]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906294372Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[2]R generation[0..12]: [174, 174, 178, 174, 178, 176, 177, 178, 169, 164, 177, 177]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.90630866Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906322629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Max found gen is 179","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906335507Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Generation requested: 180 >= found:179","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906348686Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906361585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906374283Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906394421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.90640623Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"0","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906419419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"1","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906434777Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"client":"2","":"downstairs","session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906448016Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906460284Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"382fd71d-5d8c-49a6-b198-5275b611fb8c is now active with session: 30a9e6b5-5418-4014-a152-36f35b54268b","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906472913Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906484732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912,"session_id":"30a9e6b5-5418-4014-a152-36f35b54268b"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906496891Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} {"msg":"Activated sub_volume 382fd71d-5d8c-49a6-b198-5275b611fb8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:18.906507839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 273 len 12288 data: 12 13 10 02/30 Read block 545 len 4096 03/30 Flush 04/30 Write block 435 len 40960 data: 12 15 16 21 21 16 14 12 12 12 05/30 Read block 289 len 12288 06/30 Read block 225 len 20480 07/30 Write block 254 len 36864 data: 13 12 11 14 14 15 13 12 8 08/30 Write block 426 len 8192 data: 16 19 09/30 Read block 445 len 8192 10/30 Read block 277 len 32768 11/30 Flush 12/30 Write block 508 len 36864 data: 8 11 15 14 13 11 13 11 11 13/30 Read block 80 len 40960 14/30 Write block 219 len 28672 data: 15 11 8 12 8 10 9 15/30 Read block 471 len 16384 16/30 Write block 456 len 8192 data: 16 14 17/30 Write block 74 len 16384 data: 12 15 17 17 18/30 Read block 5 len 20480 19/30 Read block 583 len 36864 20/30 Read block 192 len 24576 21/30 Read block 353 len 40960 22/30 Read block 481 len 36864 23/30 Write block 239 len 28672 data: 15 12 16 17 15 15 13 24/30 Read block 205 len 12288 25/30 Read block 524 len 12288 26/30 Read block 249 len 16384 27/30 Write block 528 len 8192 data: 12 9 28/30 Read block 528 len 40960 29/30 Write block 526 len 40960 data: 13 11 13 10 9 9 7 7 9 11 30/30 Read block 47 len 32768 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 9 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 18 19 17 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:18.928682797Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1912} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 300 300 300 F F F 1 020-039 174 174 174 300 300 300 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 174 174 174 300 300 300 T T T 4 080-099 178 178 178 308 308 308 F F F 5 100-119 176 176 176 303 303 303 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 177 177 177 307 307 307 T T T 11 220-239 177 177 177 307 307 307 T T T 12 240-259 180 180 180 310 310 310 T T T 13 260-279 180 180 180 310 310 310 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 178 178 178 308 308 308 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 178 178 178 308 308 308 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 180 180 180 310 310 310 F F F 22 440-459 180 180 180 310 310 310 T T T 23 460-479 174 174 174 300 300 300 F F F 24 480-499 178 178 178 308 308 308 F F F 25 500-519 178 178 178 308 308 308 T T T 26 520-539 176 176 176 302 302 302 T T T 27 540-559 178 178 178 308 308 308 F F F 28 560-579 178 178 178 308 308 308 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 180, Max flush: 310 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 300 300 300 F F F 1 020-039 174 174 174 300 300 300 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 181 181 181 311 311 311 F F F 4 080-099 178 178 178 308 308 308 F F F 5 100-119 176 176 176 303 303 303 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 F F F 8 160-179 169 169 169 293 293 293 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 181 181 181 311 311 311 F F F 12 240-259 181 181 181 311 311 311 F F F 13 260-279 180 180 180 310 310 310 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 178 178 178 308 308 308 F F F 17 340-359 166 166 166 288 288 288 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 178 178 178 308 308 308 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 180 180 180 310 310 310 F F F 22 440-459 181 181 181 311 311 311 F F F 23 460-479 174 174 174 300 300 300 F F F 24 480-499 178 178 178 308 308 308 F F F 25 500-519 181 181 181 311 311 311 F F F 26 520-539 181 181 181 311 311 311 F F F 27 540-559 178 178 178 308 308 308 F F F 28 560-579 178 178 178 308 308 308 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 181, Max flush: 311 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.179063159Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.222957401Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.223569228Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.223618803Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.22363967Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.223662378Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} {"msg":"Crucible b37f5796-6c39-40a2-bf18-6f08f7367d9f has session id: 3aed36d3-b127-4f28-81c5-c903f88a4965","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224224069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Upstairs opts: Upstairs UUID: b37f5796-6c39-40a2-bf18-6f08f7367d9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224259556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Crucible stats registered with UUID: b37f5796-6c39-40a2-bf18-6f08f7367d9f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224276144Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:23.224288883Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224677932Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224712018Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} {"msg":"b37f5796-6c39-40a2-bf18-6f08f7367d9f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224773202Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224800459Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"0","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224831756Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"0","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224846404Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224861663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.22488456Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"2","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224909108Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"2","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224932235Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224955703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.224980441Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"ds_connection connected from Ok(127.0.0.1:56638)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.225005768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"ds_connection connected from Ok(127.0.0.1:49056)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.225031295Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:23.225059442Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:23.22508637Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:23.22527841Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.225401806Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.225894165Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"0","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.225922092Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"2","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.226572895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"2","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.226611231Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"0","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.227156283Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:23.227264772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.22686083Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"ds_connection connected from Ok(127.0.0.1:51764)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.227033112Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.227997041Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.228570451Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[0]R flush_numbers[0..12]: [300, 300, 308, 311, 308, 303, 307, 308, 293, 284, 311, 311]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229174089Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[0]R generation[0..12]: [174, 174, 178, 181, 178, 176, 177, 178, 169, 164, 181, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229200366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229216244Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[1]R flush_numbers[0..12]: [300, 300, 308, 311, 308, 303, 307, 308, 293, 284, 311, 311]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229237922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[1]R generation[0..12]: [174, 174, 178, 181, 178, 176, 177, 178, 169, 164, 181, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229252651Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229267179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[2]R flush_numbers[0..12]: [300, 300, 308, 311, 308, 303, 307, 308, 293, 284, 311, 311]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229281177Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[2]R generation[0..12]: [174, 174, 178, 181, 178, 176, 177, 178, 169, 164, 181, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229294796Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229308705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229321403Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229334282Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229346941Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229359749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"":"downstairs","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229372798Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229384567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"0","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229397745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"1","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229410484Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"client":"2","":"downstairs","session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229423423Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229437711Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"b37f5796-6c39-40a2-bf18-6f08f7367d9f is now active with session: 3aed36d3-b127-4f28-81c5-c903f88a4965","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T05:08:33.22945071Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229471488Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920,"session_id":"3aed36d3-b127-4f28-81c5-c903f88a4965"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229483946Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} {"msg":"Activated sub_volume b37f5796-6c39-40a2-bf18-6f08f7367d9f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:33.229495115Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 11 len 36864 data: 16 17 20 22 19 19 19 17 17 03/30 Write block 454 len 40960 data: 13 14 17 15 12 11 11 11 11 9 04/30 Write block 444 len 12288 data: 13 15 15 05/30 Write block 395 len 20480 data: 13 16 14 19 19 06/30 Write block 243 len 28672 data: 16 16 14 12 12 11 11 07/30 Read block 180 len 24576 08/30 Read block 336 len 24576 09/30 Write block 65 len 4096 data: 12 10/30 Flush 11/30 Write block 166 len 28672 data: 15 13 15 16 16 18 12 12/30 Read block 198 len 36864 13/30 Read block 154 len 8192 14/30 Write block 485 len 32768 data: 11 11 10 11 11 11 9 11 15/30 Write block 448 len 12288 data: 11 14 14 16/30 Read block 344 len 36864 17/30 Flush 18/30 Write block 541 len 32768 data: 12 11 10 10 14 15 14 12 19/30 Read block 371 len 32768 20/30 Read block 554 len 40960 21/30 Write block 15 len 24576 data: 20 20 20 18 18 12 22/30 Write block 235 len 12288 data: 13 14 13 23/30 Read block 568 len 28672 24/30 Write block 270 len 4096 data: 16 25/30 Flush 26/30 Write block 341 len 36864 data: 11 13 10 10 9 7 8 11 10 27/30 Read block 290 len 36864 28/30 Write block 563 len 36864 data: 16 17 16 18 18 16 14 15 13 29/30 Write block 15 len 24576 data: 21 21 21 19 19 13 30/30 Read block 199 len 40960 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 3 4 5 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:33.250968576Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1920} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 315 315 315 T T T 1 020-039 182 182 182 315 315 315 T T T 2 040-059 178 178 178 308 308 308 F F F 3 060-079 182 182 182 313 313 313 F F F 4 080-099 178 178 178 308 308 308 F F F 5 100-119 176 176 176 303 303 303 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 F F F 8 160-179 182 182 182 314 314 314 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 182 182 182 313 313 313 F F F 13 260-279 182 182 182 315 315 315 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 178 178 178 308 308 308 F F F 17 340-359 166 166 166 288 288 288 T T T 18 360-379 173 173 173 299 299 299 F F F 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 180 180 180 310 310 310 F F F 22 440-459 182 182 182 314 314 314 F F F 23 460-479 182 182 182 313 313 313 F F F 24 480-499 182 182 182 314 314 314 F F F 25 500-519 181 181 181 311 311 311 F F F 26 520-539 181 181 181 311 311 311 F F F 27 540-559 182 182 182 315 315 315 F F F 28 560-579 178 178 178 308 308 308 T T T 29 580-599 178 178 178 308 308 308 F F F Max gen: 182, Max flush: 315 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 316 316 316 F F F 1 020-039 183 183 183 316 316 316 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 182 182 182 313 313 313 F F F 4 080-099 178 178 178 308 308 308 F F F 5 100-119 176 176 176 303 303 303 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 F F F 8 160-179 182 182 182 314 314 314 F F F 9 180-199 164 164 164 284 284 284 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 182 182 182 313 313 313 F F F 13 260-279 182 182 182 315 315 315 F F F 14 280-299 173 173 173 299 299 299 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 178 178 178 308 308 308 F F F 17 340-359 183 183 183 316 316 316 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 180 180 180 310 310 310 F F F 22 440-459 182 182 182 314 314 314 F F F 23 460-479 182 182 182 313 313 313 F F F 24 480-499 182 182 182 314 314 314 F F F 25 500-519 181 181 181 311 311 311 F F F 26 520-539 181 181 181 311 311 311 F F F 27 540-559 182 182 182 315 315 315 F F F 28 560-579 183 183 183 316 316 316 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 183, Max flush: 316 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.487641587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:37.530359222Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.530733353Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.561766117Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.562356335Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.56239977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.562418478Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.562437466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} {"msg":"Crucible 7a2ef215-c5e7-4e71-b46f-edb790a1fb71 has session id: f9f1acbb-9d75-43be-87b7-38ff1f9dcadd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563100567Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Upstairs opts: Upstairs UUID: 7a2ef215-c5e7-4e71-b46f-edb790a1fb71, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563142353Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Crucible stats registered with UUID: 7a2ef215-c5e7-4e71-b46f-edb790a1fb71","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.5631693Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:37.563189768Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563625603Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.56365023Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} {"msg":"7a2ef215-c5e7-4e71-b46f-edb790a1fb71 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563715253Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563733291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"0","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563765748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"0","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563781466Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"1","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563800534Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"1","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563824892Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"2","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563875657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"2","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563905183Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.56393514Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563963857Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"ds_connection connected from Ok(127.0.0.1:60493)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.563991824Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"ds_connection connected from Ok(127.0.0.1:55221)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.564022501Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35038)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.564153537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.564625658Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"2","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.564659585Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"1","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.564983191Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"0","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.56556728Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"1","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.565607446Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"2","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.565625384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"0","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566214122Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.56623834Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[0]R flush_numbers[0..12]: [316, 316, 308, 313, 308, 303, 307, 308, 314, 284, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566279646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[0]R generation[0..12]: [183, 183, 178, 182, 178, 176, 177, 178, 182, 164, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566298164Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566312542Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[1]R flush_numbers[0..12]: [316, 316, 308, 313, 308, 303, 307, 308, 314, 284, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566326181Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[1]R generation[0..12]: [183, 183, 178, 182, 178, 176, 177, 178, 182, 164, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.56633964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566353358Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[2]R flush_numbers[0..12]: [316, 316, 308, 313, 308, 303, 307, 308, 314, 284, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566366687Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[2]R generation[0..12]: [183, 183, 178, 182, 178, 176, 177, 178, 182, 164, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566380105Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566393864Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566411622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566425291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566438069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566450668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"":"downstairs","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:37.566463847Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566478985Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"0","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566492593Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"1","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566505952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"client":"2","":"downstairs","session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566519551Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.56653182Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"7a2ef215-c5e7-4e71-b46f-edb790a1fb71 is now active with session: f9f1acbb-9d75-43be-87b7-38ff1f9dcadd","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566544668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566556427Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928,"session_id":"f9f1acbb-9d75-43be-87b7-38ff1f9dcadd"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566568386Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} {"msg":"Activated sub_volume 7a2ef215-c5e7-4e71-b46f-edb790a1fb71","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:37.566579484Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 267 len 36864 data: 14 17 18 17 16 14 13 14 11 02/30 Write block 512 len 20480 data: 14 12 14 12 12 03/30 Read block 410 len 4096 04/30 Read block 419 len 16384 05/30 Write block 108 len 40960 data: 14 13 13 16 16 12 11 17 15 13 06/30 Read block 588 len 12288 07/30 Write block 75 len 28672 data: 16 18 18 17 19 18 17 08/30 Read block 100 len 32768 09/30 Read block 517 len 4096 10/30 Write block 521 len 36864 data: 10 14 16 16 15 14 12 14 11 11/30 Read block 426 len 28672 12/30 Write block 275 len 8192 data: 12 11 13/30 Read block 66 len 16384 14/30 Write block 439 len 12288 data: 22 17 15 15/30 Write block 182 len 12288 data: 12 12 17 16/30 Read block 563 len 24576 17/30 Read block 22 len 40960 18/30 Read block 187 len 40960 19/30 Read block 524 len 16384 20/30 Read block 425 len 12288 21/30 Write block 176 len 32768 data: 20 20 17 13 10 13 13 13 22/30 Write block 13 len 12288 data: 21 23 22 23/30 Flush 24/30 Write block 247 len 24576 data: 13 12 12 8 9 10 25/30 Write block 285 len 40960 data: 16 17 19 20 18 21 23 22 18 18 26/30 Write block 462 len 40960 data: 12 10 12 10 9 11 11 9 8 11 27/30 Write block 65 len 36864 data: 13 9 8 8 7 7 7 7 10 28/30 Read block 523 len 32768 29/30 Write block 332 len 28672 data: 10 7 9 7 8 11 12 30/30 Read block 461 len 40960 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:37.588423646Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 317 317 317 F F F 1 020-039 183 183 183 316 316 316 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 184 184 184 317 317 317 T T T 4 080-099 184 184 184 317 317 317 F F F 5 100-119 184 184 184 317 317 317 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 184 184 184 317 317 317 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 182 182 182 313 313 313 T T T 13 260-279 184 184 184 317 317 317 F F F 14 280-299 173 173 173 299 299 299 T T T 15 300-319 176 176 176 304 304 304 F F F 16 320-339 178 178 178 308 308 308 T T T 17 340-359 183 183 183 316 316 316 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 184 184 184 317 317 317 F F F 22 440-459 184 184 184 317 317 317 F F F 23 460-479 182 182 182 313 313 313 T T T 24 480-499 182 182 182 314 314 314 F F F 25 500-519 184 184 184 317 317 317 F F F 26 520-539 184 184 184 317 317 317 F F F 27 540-559 182 182 182 315 315 315 F F F 28 560-579 183 183 183 316 316 316 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 184, Max flush: 317 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 317 317 317 F F F 1 020-039 183 183 183 316 316 316 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 185 185 185 318 318 318 F F F 4 080-099 184 184 184 317 317 317 F F F 5 100-119 184 184 184 317 317 317 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 184 184 184 317 317 317 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 185 185 185 318 318 318 F F F 13 260-279 184 184 184 317 317 317 F F F 14 280-299 185 185 185 318 318 318 F F F 15 300-319 176 176 176 304 304 304 F F F 16 320-339 185 185 185 318 318 318 F F F 17 340-359 183 183 183 316 316 316 F F F 18 360-379 173 173 173 299 299 299 F F F 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 184 184 184 317 317 317 F F F 22 440-459 184 184 184 317 317 317 F F F 23 460-479 185 185 185 318 318 318 F F F 24 480-499 182 182 182 314 314 314 F F F 25 500-519 184 184 184 317 317 317 F F F 26 520-539 184 184 184 317 317 317 F F F 27 540-559 182 182 182 315 315 315 F F F 28 560-579 183 183 183 316 316 316 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 185, Max flush: 318 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.856843886Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.902021384Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.902622962Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.902667107Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.902683495Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.902698544Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} {"msg":"Crucible a190fe03-eb7d-4bf6-8238-bbc77b31c7ef has session id: 328b7a1f-54fe-481b-8d59-fee5e684014f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.903206371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Upstairs opts: Upstairs UUID: a190fe03-eb7d-4bf6-8238-bbc77b31c7ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.903276733Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Crucible stats registered with UUID: a190fe03-eb7d-4bf6-8238-bbc77b31c7ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.903297751Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:41.903318619Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.903764313Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.9037881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} {"msg":"a190fe03-eb7d-4bf6-8238-bbc77b31c7ef active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.903865412Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.903901179Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"0","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.903950473Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"0","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.90397947Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"1","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904029505Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"1","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904063431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"2","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904093518Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"2","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904119926Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904148633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904343692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37782)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904374809Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56622)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904401126Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55381)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904416545Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904905724Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"0","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.904960978Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"1","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.905410481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"2","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.905636667Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"1","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.905854815Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"0","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906046285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"2","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906215187Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906295569Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[0]R flush_numbers[0..12]: [317, 316, 308, 318, 317, 317, 307, 308, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.90657302Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[0]R generation[0..12]: [184, 183, 178, 185, 184, 184, 177, 178, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906597587Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906612676Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[1]R flush_numbers[0..12]: [317, 316, 308, 318, 317, 317, 307, 308, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906630044Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[1]R generation[0..12]: [184, 183, 178, 185, 184, 184, 177, 178, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906644562Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906658621Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[2]R flush_numbers[0..12]: [317, 316, 308, 318, 317, 317, 307, 308, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.90667231Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[2]R generation[0..12]: [184, 183, 178, 185, 184, 184, 177, 178, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906686078Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906699927Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906712725Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906725784Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906738592Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906751421Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906774299Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906785998Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"0","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906799116Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"1","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906811835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"client":"2","":"downstairs","session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906824694Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906839332Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"a190fe03-eb7d-4bf6-8238-bbc77b31c7ef is now active with session: 328b7a1f-54fe-481b-8d59-fee5e684014f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906852231Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.90686417Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936,"session_id":"328b7a1f-54fe-481b-8d59-fee5e684014f"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906876178Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} {"msg":"Activated sub_volume a190fe03-eb7d-4bf6-8238-bbc77b31c7ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:41.906887287Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 102 len 20480 data: 14 11 8 10 11 02/30 Read block 47 len 4096 03/30 Flush 04/30 Write block 255 len 28672 data: 13 12 15 15 16 14 13 05/30 Read block 442 len 24576 06/30 Flush 07/30 Write block 157 len 12288 data: 14 17 18 08/30 Write block 16 len 16384 data: 22 22 20 20 09/30 Write block 306 len 32768 data: 16 15 15 15 18 17 16 16 10/30 Read block 268 len 28672 11/30 Read block 367 len 12288 12/30 Read block 202 len 40960 13/30 Read block 521 len 20480 14/30 Read block 198 len 4096 15/30 Write block 77 len 16384 data: 19 18 20 19 16/30 Read block 306 len 16384 17/30 Write block 352 len 16384 data: 16 18 14 15 18/30 Write block 361 len 20480 data: 10 9 9 11 12 19/30 Read block 157 len 16384 20/30 Read block 14 len 12288 21/30 Write block 483 len 24576 data: 10 10 12 12 11 12 22/30 Read block 388 len 8192 23/30 Write block 447 len 24576 data: 15 12 15 15 13 15 24/30 Read block 360 len 12288 25/30 Write block 517 len 28672 data: 11 10 8 9 11 15 17 26/30 Read block 434 len 32768 27/30 Write block 435 len 32768 data: 13 16 17 22 23 18 16 13 28/30 Write block 249 len 24576 data: 13 9 10 11 10 14 29/30 Read block 120 len 28672 30/30 Read block 463 len 40960 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 3 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 23 24 23 70 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:41.927815954Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1936} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 317 317 317 T T T 1 020-039 183 183 183 316 316 316 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 185 185 185 318 318 318 T T T 4 080-099 184 184 184 317 317 317 T T T 5 100-119 186 186 186 319 319 319 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 178 178 178 308 308 308 T T T 8 160-179 184 184 184 317 317 317 F F F 9 180-199 184 184 184 317 317 317 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 186 186 186 320 320 320 T T T 13 260-279 186 186 186 320 320 320 F F F 14 280-299 185 185 185 318 318 318 F F F 15 300-319 176 176 176 304 304 304 T T T 16 320-339 185 185 185 318 318 318 F F F 17 340-359 183 183 183 316 316 316 T T T 18 360-379 173 173 173 299 299 299 T T T 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 184 184 184 317 317 317 T T T 22 440-459 184 184 184 317 317 317 T T T 23 460-479 185 185 185 318 318 318 F F F 24 480-499 182 182 182 314 314 314 T T T 25 500-519 184 184 184 317 317 317 T T T 26 520-539 184 184 184 317 317 317 T T T 27 540-559 182 182 182 315 315 315 F F F 28 560-579 183 183 183 316 316 316 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 186, Max flush: 320 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 321 321 321 F F F 1 020-039 183 183 183 316 316 316 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 187 187 187 321 321 321 F F F 4 080-099 187 187 187 321 321 321 F F F 5 100-119 186 186 186 319 319 319 F F F 6 120-139 177 177 177 307 307 307 F F F 7 140-159 187 187 187 321 321 321 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 184 184 184 317 317 317 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 187 187 187 321 321 321 F F F 13 260-279 186 186 186 320 320 320 F F F 14 280-299 185 185 185 318 318 318 F F F 15 300-319 187 187 187 321 321 321 F F F 16 320-339 185 185 185 318 318 318 F F F 17 340-359 187 187 187 321 321 321 F F F 18 360-379 187 187 187 321 321 321 F F F 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 187 187 187 321 321 321 F F F 22 440-459 187 187 187 321 321 321 F F F 23 460-479 185 185 185 318 318 318 F F F 24 480-499 187 187 187 321 321 321 F F F 25 500-519 187 187 187 321 321 321 F F F 26 520-539 187 187 187 321 321 321 F F F 27 540-559 182 182 182 315 315 315 F F F 28 560-579 183 183 183 316 316 316 F F F 29 580-599 178 178 178 308 308 308 F F F Max gen: 187, Max flush: 321 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.207667902Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:46.250382657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.250780556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.28152155Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.282092041Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.282126937Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.282141765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.282156284Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} {"msg":"Crucible 31c88b3f-2827-4192-ba1d-cac8973ba971 has session id: 8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.282694788Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Upstairs opts: Upstairs UUID: 31c88b3f-2827-4192-ba1d-cac8973ba971, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.282733044Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Crucible stats registered with UUID: 31c88b3f-2827-4192-ba1d-cac8973ba971","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.282758901Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:46.282780399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283204375Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283228582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} {"msg":"31c88b3f-2827-4192-ba1d-cac8973ba971 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283294165Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283312003Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"0","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.28334012Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"0","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283354399Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"1","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283375377Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"1","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283400884Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"2","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283427931Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"2","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283453249Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283480536Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283510553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"ds_connection connected from Ok(127.0.0.1:48267)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.28353629Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"ds_connection connected from Ok(127.0.0.1:64038)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283563187Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"ds_connection connected from Ok(127.0.0.1:59172)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.283588155Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.28420783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"1","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.284256495Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"2","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.2844002Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"0","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.284941513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"2","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.28506844Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"1","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285281008Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"0","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285542481Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285636221Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[0]R flush_numbers[0..12]: [321, 316, 308, 321, 321, 319, 307, 321, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285791885Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[0]R generation[0..12]: [187, 183, 178, 187, 187, 186, 177, 187, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285813453Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285829521Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[1]R flush_numbers[0..12]: [321, 316, 308, 321, 321, 319, 307, 321, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285844129Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[1]R generation[0..12]: [187, 183, 178, 187, 187, 186, 177, 187, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285858328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285874196Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[2]R flush_numbers[0..12]: [321, 316, 308, 321, 321, 319, 307, 321, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285888245Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"[2]R generation[0..12]: [187, 183, 178, 187, 187, 186, 177, 187, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285905393Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285926461Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285940419Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285954568Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285967957Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285980705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"":"downstairs","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.285993964Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286006313Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"0","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286019291Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"1","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.28603218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"client":"2","":"downstairs","session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286045238Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286057757Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"31c88b3f-2827-4192-ba1d-cac8973ba971 is now active with session: 8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286071006Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286083445Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944,"session_id":"8b6ad3f4-dfda-4be7-a3a2-8f3772c19b25"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286096153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} {"msg":"Activated sub_volume 31c88b3f-2827-4192-ba1d-cac8973ba971","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:46.286109932Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 109 len 32768 data: 14 14 17 17 13 12 18 16 02/30 Write block 496 len 40960 data: 11 12 13 14 14 15 19 16 16 15 03/30 Write block 558 len 28672 data: 14 15 14 15 15 17 18 04/30 Read block 560 len 16384 05/30 Read block 8 len 4096 06/30 Read block 146 len 40960 07/30 Write block 508 len 24576 data: 9 12 16 15 15 13 08/30 Read block 125 len 32768 09/30 Write block 326 len 36864 data: 10 10 12 12 11 12 11 8 10 10/30 Read block 70 len 12288 11/30 Read block 411 len 40960 12/30 Read block 462 len 24576 13/30 Read block 420 len 16384 14/30 Write block 324 len 28672 data: 12 11 11 11 13 13 12 15/30 Flush 16/30 Write block 91 len 4096 data: 18 17/30 Read block 268 len 20480 18/30 Read block 516 len 12288 19/30 Write block 579 len 4096 data: 15 20/30 Write block 358 len 8192 data: 14 10 21/30 Read block 16 len 20480 22/30 Write block 17 len 12288 data: 23 21 21 23/30 Read block 417 len 36864 24/30 Write block 326 len 28672 data: 12 12 14 14 13 13 12 25/30 Write block 62 len 4096 data: 16 26/30 Read block 151 len 32768 27/30 Write block 133 len 8192 data: 10 10 28/30 Write block 337 len 40960 data: 12 13 10 11 12 14 11 11 10 8 29/30 Read block 487 len 24576 30/30 Write block 576 len 40960 data: 15 14 16 16 10 12 11 12 14 15 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 1 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:46.307247257Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1944} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 321 321 321 T T T 1 020-039 183 183 183 316 316 316 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 187 187 187 321 321 321 T T T 4 080-099 187 187 187 321 321 321 T T T 5 100-119 188 188 188 322 322 322 F F F 6 120-139 177 177 177 307 307 307 T T T 7 140-159 187 187 187 321 321 321 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 184 184 184 317 317 317 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 187 187 187 321 321 321 F F F 13 260-279 186 186 186 320 320 320 F F F 14 280-299 185 185 185 318 318 318 F F F 15 300-319 187 187 187 321 321 321 F F F 16 320-339 188 188 188 322 322 322 T T T 17 340-359 187 187 187 321 321 321 T T T 18 360-379 187 187 187 321 321 321 F F F 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 187 187 187 321 321 321 F F F 22 440-459 187 187 187 321 321 321 F F F 23 460-479 185 185 185 318 318 318 F F F 24 480-499 188 188 188 322 322 322 F F F 25 500-519 188 188 188 322 322 322 F F F 26 520-539 187 187 187 321 321 321 F F F 27 540-559 188 188 188 322 322 322 F F F 28 560-579 188 188 188 322 322 322 T T T 29 580-599 178 178 178 308 308 308 T T T Max gen: 188, Max flush: 322 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 323 323 323 F F F 1 020-039 183 183 183 316 316 316 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 189 189 189 323 323 323 F F F 4 080-099 189 189 189 323 323 323 F F F 5 100-119 188 188 188 322 322 322 F F F 6 120-139 189 189 189 323 323 323 F F F 7 140-159 187 187 187 321 321 321 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 184 184 184 317 317 317 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 187 187 187 321 321 321 F F F 13 260-279 186 186 186 320 320 320 F F F 14 280-299 185 185 185 318 318 318 F F F 15 300-319 187 187 187 321 321 321 F F F 16 320-339 189 189 189 323 323 323 F F F 17 340-359 189 189 189 323 323 323 F F F 18 360-379 187 187 187 321 321 321 F F F 19 380-399 182 182 182 313 313 313 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 187 187 187 321 321 321 F F F 22 440-459 187 187 187 321 321 321 F F F 23 460-479 185 185 185 318 318 318 F F F 24 480-499 188 188 188 322 322 322 F F F 25 500-519 188 188 188 322 322 322 F F F 26 520-539 187 187 187 321 321 321 F F F 27 540-559 188 188 188 322 322 322 F F F 28 560-579 189 189 189 323 323 323 F F F 29 580-599 189 189 189 323 323 323 F F F Max gen: 189, Max flush: 323 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.590477772Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.635291699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.635976867Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.636028152Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.636060439Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.636088916Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} {"msg":"Crucible ea918fa9-cf83-4f2b-8ad9-ddfdde339c63 has session id: b82ebea4-3478-4b7b-9c43-3743f2353379","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.636650137Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Upstairs opts: Upstairs UUID: ea918fa9-cf83-4f2b-8ad9-ddfdde339c63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.636695663Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Crucible stats registered with UUID: ea918fa9-cf83-4f2b-8ad9-ddfdde339c63","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.63672012Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:50.636735478Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637166254Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637189371Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} {"msg":"ea918fa9-cf83-4f2b-8ad9-ddfdde339c63 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637265023Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637283391Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"0","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637311368Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"0","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637325277Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"1","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637339765Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"1","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637353334Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"2","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637371922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"2","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637385571Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637399359Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637412888Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"ds_connection connected from Ok(127.0.0.1:51316)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637508288Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"ds_connection connected from Ok(127.0.0.1:52183)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637714896Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"ds_connection connected from Ok(127.0.0.1:56711)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.637756952Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.638296516Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"0","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.638337781Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"1","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.638370568Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"2","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.63892767Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"1","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639071005Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"0","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639158556Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"2","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639439526Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639633076Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[0]R flush_numbers[0..12]: [323, 316, 308, 323, 323, 322, 323, 321, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639773162Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[0]R generation[0..12]: [189, 183, 178, 189, 189, 188, 189, 187, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639801919Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639817647Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[1]R flush_numbers[0..12]: [323, 316, 308, 323, 323, 322, 323, 321, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639831605Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[1]R generation[0..12]: [189, 183, 178, 189, 189, 188, 189, 187, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639845484Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639859673Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[2]R flush_numbers[0..12]: [323, 316, 308, 323, 323, 322, 323, 321, 317, 317, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639873391Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[2]R generation[0..12]: [189, 183, 178, 189, 189, 188, 189, 187, 184, 184, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.63988703Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639900818Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639913657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639940474Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Next flush: 324","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2025-03-18T05:08:50.639960282Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639973301Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"":"downstairs","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.639990889Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640003448Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"0","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640016706Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"1","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640029525Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"client":"2","":"downstairs","session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640042474Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640054762Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"ea918fa9-cf83-4f2b-8ad9-ddfdde339c63 is now active with session: b82ebea4-3478-4b7b-9c43-3743f2353379","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640067411Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.64007924Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952,"session_id":"b82ebea4-3478-4b7b-9c43-3743f2353379"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640091369Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} {"msg":"Activated sub_volume ea918fa9-cf83-4f2b-8ad9-ddfdde339c63","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:50.640102457Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 28672 data: 20 19 19 16 14 13 13 02/30 Write block 180 len 28672 data: 11 14 14 14 18 15 16 03/30 Write block 463 len 8192 data: 11 13 04/30 Read block 267 len 4096 05/30 Read block 458 len 16384 06/30 Read block 380 len 8192 07/30 Read block 465 len 16384 08/30 Write block 243 len 20480 data: 17 17 15 13 14 09/30 Read block 384 len 12288 10/30 Write block 289 len 12288 data: 19 22 24 11/30 Write block 471 len 40960 data: 12 12 15 12 13 14 19 17 17 14 12/30 Read block 24 len 20480 13/30 Write block 390 len 20480 data: 9 12 15 15 16 14/30 Flush 15/30 Write block 547 len 20480 data: 15 13 10 10 10 16/30 Read block 269 len 4096 17/30 Write block 498 len 40960 data: 14 15 15 16 20 17 17 16 14 12 18/30 Write block 16 len 36864 data: 23 24 22 22 14 9 11 10 10 19/30 Read block 145 len 28672 20/30 Read block 587 len 20480 21/30 Read block 245 len 12288 22/30 Flush 23/30 Write block 81 len 12288 data: 18 18 22 24/30 Write block 331 len 40960 data: 14 13 9 11 8 9 13 14 11 12 25/30 Read block 8 len 12288 26/30 Flush 27/30 Write block 302 len 8192 data: 19 19 28/30 Write block 546 len 24576 data: 16 16 14 11 11 11 29/30 Write block 545 len 4096 data: 15 30/30 Read block 117 len 12288 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:50.660047527Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1952} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 189 189 189 323 323 323 F F F 4 080-099 190 190 190 326 326 326 F F F 5 100-119 188 188 188 322 322 322 F F F 6 120-139 189 189 189 323 323 323 F F F 7 140-159 187 187 187 321 321 321 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 190 190 190 324 324 324 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 190 190 190 324 324 324 F F F 13 260-279 186 186 186 320 320 320 F F F 14 280-299 190 190 190 324 324 324 F F F 15 300-319 187 187 187 321 321 321 T T T 16 320-339 190 190 190 326 326 326 F F F 17 340-359 190 190 190 326 326 326 F F F 18 360-379 187 187 187 321 321 321 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 187 187 187 321 321 321 F F F 22 440-459 187 187 187 321 321 321 F F F 23 460-479 190 190 190 324 324 324 F F F 24 480-499 190 190 190 325 325 325 F F F 25 500-519 190 190 190 325 325 325 F F F 26 520-539 187 187 187 321 321 321 F F F 27 540-559 190 190 190 325 325 325 T T T 28 560-579 189 189 189 323 323 323 F F F 29 580-599 189 189 189 323 323 323 F F F Max gen: 190, Max flush: 326 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 178 178 178 308 308 308 F F F 3 060-079 189 189 189 323 323 323 F F F 4 080-099 190 190 190 326 326 326 F F F 5 100-119 188 188 188 322 322 322 F F F 6 120-139 189 189 189 323 323 323 F F F 7 140-159 187 187 187 321 321 321 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 190 190 190 324 324 324 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 F F F 12 240-259 190 190 190 324 324 324 F F F 13 260-279 186 186 186 320 320 320 F F F 14 280-299 190 190 190 324 324 324 F F F 15 300-319 191 191 191 327 327 327 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 190 190 190 326 326 326 F F F 18 360-379 187 187 187 321 321 321 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 178 178 178 308 308 308 F F F 21 420-439 187 187 187 321 321 321 F F F 22 440-459 187 187 187 321 321 321 F F F 23 460-479 190 190 190 324 324 324 F F F 24 480-499 190 190 190 325 325 325 F F F 25 500-519 190 190 190 325 325 325 F F F 26 520-539 187 187 187 321 321 321 F F F 27 540-559 191 191 191 327 327 327 F F F 28 560-579 189 189 189 323 323 323 F F F 29 580-599 189 189 189 323 323 323 F F F Max gen: 191, Max flush: 327 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.882130499Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:54.925419295Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.925800575Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.957265634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.957827295Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.957867131Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.957883609Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.957901157Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} {"msg":"Crucible 72ea3138-97e6-436d-ac9d-e32c0abf7d4a has session id: b7ede52c-9f1e-423f-ab2b-4107af73075f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.95854536Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Upstairs opts: Upstairs UUID: 72ea3138-97e6-436d-ac9d-e32c0abf7d4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.958597685Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Crucible stats registered with UUID: 72ea3138-97e6-436d-ac9d-e32c0abf7d4a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.958624922Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:54.95864105Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.958986284Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959011762Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} {"msg":"72ea3138-97e6-436d-ac9d-e32c0abf7d4a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959062536Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959089634Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"0","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.9591282Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"0","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959156667Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"1","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959186554Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"1","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.95921524Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"2","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959245877Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"2","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959271225Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959293292Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.95931769Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41616)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959343837Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57248)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959370704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50121)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.959491582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.960092699Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"1","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.960132345Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"2","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.960388748Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"0","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.960923992Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"1","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961014493Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"2","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961049279Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"0","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961510971Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961537288Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[0]R flush_numbers[0..12]: [325, 325, 308, 323, 326, 322, 323, 321, 317, 324, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961564186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[0]R generation[0..12]: [190, 190, 178, 189, 190, 188, 189, 187, 184, 190, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961579484Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961594063Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[1]R flush_numbers[0..12]: [325, 325, 308, 323, 326, 322, 323, 321, 317, 324, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961608221Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[1]R generation[0..12]: [190, 190, 178, 189, 190, 188, 189, 187, 184, 190, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961642607Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[2]R flush_numbers[0..12]: [325, 325, 308, 323, 326, 322, 323, 321, 317, 324, 311, 315]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961660705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[2]R generation[0..12]: [190, 190, 178, 189, 190, 188, 189, 187, 184, 190, 181, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961674734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961688522Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961701081Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.96171405Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961726749Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961739597Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"":"downstairs","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T05:08:54.961753186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961771134Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"0","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961784392Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"1","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961797321Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"client":"2","":"downstairs","session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.96181061Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961823328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"72ea3138-97e6-436d-ac9d-e32c0abf7d4a is now active with session: b7ede52c-9f1e-423f-ab2b-4107af73075f","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961836167Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961848046Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960,"session_id":"b7ede52c-9f1e-423f-ab2b-4107af73075f"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961860045Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} {"msg":"Activated sub_volume 72ea3138-97e6-436d-ac9d-e32c0abf7d4a","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:54.961873783Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 40960 data: 15 13 13 12 11 9 10 12 16 18 02/30 Write block 364 len 16384 data: 12 13 12 12 03/30 Read block 58 len 16384 04/30 Read block 77 len 4096 05/30 Read block 235 len 32768 06/30 Read block 554 len 4096 07/30 Flush 08/30 Flush 09/30 Write block 358 len 4096 data: 15 10/30 Write block 57 len 24576 data: 16 20 24 21 18 17 11/30 Read block 526 len 16384 12/30 Read block 89 len 8192 13/30 Write block 456 len 4096 data: 18 14/30 Write block 531 len 16384 data: 10 8 8 10 15/30 Write block 589 len 28672 data: 13 14 11 9 12 10 7 16/30 Write block 190 len 40960 data: 15 16 15 15 17 17 19 16 13 14 17/30 Write block 147 len 36864 data: 14 14 12 10 6 11 11 11 11 18/30 Read block 489 len 4096 19/30 Read block 587 len 12288 20/30 Write block 470 len 40960 data: 9 13 13 16 13 14 15 20 18 18 21/30 Read block 254 len 8192 22/30 Read block 506 len 36864 23/30 Write block 416 len 20480 data: 12 14 15 13 11 24/30 Write block 527 len 36864 data: 13 15 12 10 11 9 9 11 12 25/30 Read block 545 len 24576 26/30 Write block 142 len 16384 data: 11 11 12 12 27/30 Write block 569 len 28672 data: 15 16 14 19 22 22 22 28/30 Read block 181 len 16384 29/30 Write block 221 len 20480 data: 9 13 9 11 10 30/30 Write block 357 len 8192 data: 14 16 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 1 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:54.981497097Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1960} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 178 178 178 308 308 308 T T T 3 060-079 189 189 189 323 323 323 T T T 4 080-099 190 190 190 326 326 326 F F F 5 100-119 188 188 188 322 322 322 F F F 6 120-139 189 189 189 323 323 323 F F F 7 140-159 187 187 187 321 321 321 T T T 8 160-179 184 184 184 317 317 317 F F F 9 180-199 190 190 190 324 324 324 T T T 10 200-219 181 181 181 311 311 311 F F F 11 220-239 182 182 182 315 315 315 T T T 12 240-259 190 190 190 324 324 324 F F F 13 260-279 186 186 186 320 320 320 F F F 14 280-299 190 190 190 324 324 324 F F F 15 300-319 191 191 191 327 327 327 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 190 190 190 326 326 326 T T T 18 360-379 192 192 192 328 328 328 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 178 178 178 308 308 308 T T T 21 420-439 187 187 187 321 321 321 T T T 22 440-459 187 187 187 321 321 321 T T T 23 460-479 190 190 190 324 324 324 T T T 24 480-499 190 190 190 325 325 325 F F F 25 500-519 192 192 192 328 328 328 F F F 26 520-539 192 192 192 328 328 328 T T T 27 540-559 191 191 191 327 327 327 F F F 28 560-579 189 189 189 323 323 323 T T T 29 580-599 189 189 189 323 323 323 T T T Max gen: 192, Max flush: 328 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 193 193 193 329 329 329 F F F 3 060-079 193 193 193 329 329 329 F F F 4 080-099 190 190 190 326 326 326 F F F 5 100-119 188 188 188 322 322 322 F F F 6 120-139 189 189 189 323 323 323 F F F 7 140-159 193 193 193 329 329 329 F F F 8 160-179 184 184 184 317 317 317 F F F 9 180-199 193 193 193 329 329 329 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 193 193 193 329 329 329 F F F 12 240-259 190 190 190 324 324 324 F F F 13 260-279 186 186 186 320 320 320 F F F 14 280-299 190 190 190 324 324 324 F F F 15 300-319 191 191 191 327 327 327 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 193 193 193 329 329 329 F F F 18 360-379 192 192 192 328 328 328 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 193 193 193 329 329 329 F F F 21 420-439 193 193 193 329 329 329 F F F 22 440-459 193 193 193 329 329 329 F F F 23 460-479 193 193 193 329 329 329 F F F 24 480-499 190 190 190 325 325 325 F F F 25 500-519 192 192 192 328 328 328 F F F 26 520-539 193 193 193 329 329 329 F F F 27 540-559 191 191 191 327 327 327 F F F 28 560-579 193 193 193 329 329 329 F F F 29 580-599 193 193 193 329 329 329 F F F Max gen: 193, Max flush: 329 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.288685683Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.33254742Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.333174214Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.33321166Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.333226859Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.333241337Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} {"msg":"Crucible 719bb20a-ab0c-4721-849b-99a408303afb has session id: eee6dde6-f243-4ab0-b421-d224594e1333","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.33378553Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Upstairs opts: Upstairs UUID: 719bb20a-ab0c-4721-849b-99a408303afb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.333823256Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Crucible stats registered with UUID: 719bb20a-ab0c-4721-849b-99a408303afb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.333839995Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:59.333853233Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334294817Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334319705Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} {"msg":"719bb20a-ab0c-4721-849b-99a408303afb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334400056Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334417974Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"0","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334447081Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"0","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.33446178Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"1","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334480068Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"1","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334493406Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"2","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334507845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"2","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334525763Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334543611Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.33455692Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"ds_connection connected from Ok(127.0.0.1:35799)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.3347495Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"ds_connection connected from Ok(127.0.0.1:37130)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334901354Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"ds_connection connected from Ok(127.0.0.1:42541)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.334921422Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.335666524Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"0","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.335692872Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"1","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.336314537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"0","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.33647779Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"1","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.336915934Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.337088296Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.337170797Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"2","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.337699802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"2","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[0]R flush_numbers[0..12]: [325, 325, 329, 329, 326, 322, 323, 329, 317, 329, 311, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338226977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[0]R generation[0..12]: [190, 190, 193, 193, 190, 188, 189, 193, 184, 193, 181, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338251055Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338269153Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[1]R flush_numbers[0..12]: [325, 325, 329, 329, 326, 322, 323, 329, 317, 329, 311, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338286661Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[1]R generation[0..12]: [190, 190, 193, 193, 190, 188, 189, 193, 184, 193, 181, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338303739Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338321137Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[2]R flush_numbers[0..12]: [325, 325, 329, 329, 326, 322, 323, 329, 317, 329, 311, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338337985Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[2]R generation[0..12]: [190, 190, 193, 193, 190, 188, 189, 193, 184, 193, 181, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338354734Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338371732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.3383876Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB",Wait for a query_work_queue command to finish before sending IO "pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338411308Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338432136Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338448544Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"":"downstairs","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338464732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338479431Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"0","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.33849246Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"1","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338505668Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"client":"2","":"downstairs","session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338519047Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338531076Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"719bb20a-ab0c-4721-849b-99a408303afb is now active with session: eee6dde6-f243-4ab0-b421-d224594e1333","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338543774Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338555433Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968,"session_id":"eee6dde6-f243-4ab0-b421-d224594e1333"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338567462Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} {"msg":"Activated sub_volume 719bb20a-ab0c-4721-849b-99a408303afb","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:08:59.338578511Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 138 len 36864 data: 11 13 13 14 12 12 13 13 12 02/30 Write block 146 len 36864 data: 13 15 15 13 11 7 12 12 12 03/30 Read block 348 len 12288 04/30 Write block 159 len 4096 data: 19 05/30 Read block 523 len 36864 06/30 Write block 419 len 16384 data: 14 12 10 8 07/30 Write block 582 len 16384 data: 12 13 15 16 08/30 Flush 09/30 Flush 10/30 Write block 357 len 16384 data: 15 17 11 9 11/30 Read block 495 len 28672 12/30 Read block 106 len 32768 13/30 Read block 36 len 32768 14/30 Write block 459 len 40960 data: 12 12 12 13 12 14 11 10 12 12 15/30 Write block 151 len 40960 data: 8 13 13 13 12 11 15 18 20 20 16/30 Write block 40 len 28672 data: 18 22 19 15 14 11 11 17/30 Write block 583 len 20480 data: 14 16 17 14 15 18/30 Read block 326 len 36864 19/30 Write block 311 len 16384 data: 18 17 17 17 20/30 Read block 228 len 40960 21/30 Write block 405 len 4096 data: 14 22/30 Read block 564 len 28672 23/30 Flush 24/30 Write block 264 len 24576 data: 10 13 14 15 18 19 25/30 Write block 47 len 32768 data: 11 10 11 14 14 18 13 13 26/30 Read block 464 len 40960 27/30 Read block 529 len 32768 28/30 Write block 450 len 36864 data: 16 14 16 15 14 15 19 16 13 29/30 Read block 397 len 16384 30/30 Read block 421 len 4096 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:08:59.359722186Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1968} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 194 194 194 332 332 332 T T T 3 060-079 193 193 193 329 329 329 F F F 4 080-099 190 190 190 326 326 326 F F F 5 100-119 188 188 188 322 322 322 F F F 6 120-139 194 194 194 330 330 330 F F F 7 140-159 194 194 194 332 332 332 F F F 8 160-179 194 194 194 332 332 332 F F F 9 180-199 193 193 193 329 329 329 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 193 193 193 329 329 329 F F F 12 240-259 190 190 190 324 324 324 F F F 13 260-279 186 186 186 320 320 320 T T T 14 280-299 190 190 190 324 324 324 F F F 15 300-319 194 194 194 332 332 332 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 194 194 194 332 332 332 F F F 18 360-379 194 194 194 332 332 332 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 194 194 194 332 332 332 F F F 21 420-439 194 194 194 330 330 330 F F F 22 440-459 194 194 194 332 332 332 T T T 23 460-479 194 194 194 332 332 332 F F F 24 480-499 190 190 190 325 325 325 F F F 25 500-519 192 192 192 328 328 328 F F F 26 520-539 193 193 193 329 329 329 F F F 27 540-559 191 191 191 327 327 327 F F F 28 560-579 193 193 193 329 329 329 F F F 29 580-599 194 194 194 332 332 332 F F F Max gen: 194, Max flush: 332 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 195 195 195 333 333 333 F F F 3 060-079 193 193 193 329 329 329 F F F 4 080-099 190 190 190 326 326 326 F F F 5 100-119 188 188 188 322 322 322 F F F 6 120-139 194 194 194 330 330 330 F F F 7 140-159 194 194 194 332 332 332 F F F 8 160-179 194 194 194 332 332 332 F F F 9 180-199 193 193 193 329 329 329 F F F 10 200-219 181 181 181 311 311 311 F F F 11 220-239 193 193 193 329 329 329 F F F 12 240-259 190 190 190 324 324 324 F F F 13 260-279 195 195 195 333 333 333 F F F 14 280-299 190 190 190 324 324 324 F F F 15 300-319 194 194 194 332 332 332 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 194 194 194 332 332 332 F F F 18 360-379 194 194 194 332 332 332 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 194 194 194 332 332 332 F F F 21 420-439 194 194 194 330 330 330 F F F 22 440-459 195 195 195 333 333 333 F F F 23 460-479 194 194 194 332 332 332 F F F 24 480-499 190 190 190 325 325 325 F F F 25 500-519 192 192 192 328 328 328 F F F 26 520-539 193 193 193 329 329 329 F F F 27 540-559 191 191 191 327 327 327 F F F 28 560-579 193 193 193 329 329 329 F F F 29 580-599 194 194 194 332 332 332 F F F Max gen: 195, Max flush: 333 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.586025218Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.630601269Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.631280329Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.631314405Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.631328453Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.631342622Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} {"msg":"Crucible 20b05080-0ad4-400f-a643-8280f016a4ff has session id: 58e31f0e-494a-460d-85aa-bab8befa6e28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.631907893Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Upstairs opts: Upstairs UUID: 20b05080-0ad4-400f-a643-8280f016a4ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.631949539Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Crucible stats registered with UUID: 20b05080-0ad4-400f-a643-8280f016a4ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.631967747Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:03.631984975Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632532828Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632565994Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} {"msg":"20b05080-0ad4-400f-a643-8280f016a4ff active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632621229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632648226Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"0","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632678523Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"0","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632693691Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"1","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632708069Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"1","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632724718Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"2","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632749835Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"2","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632773293Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.63279672Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632822738Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"ds_connection connected from Ok(127.0.0.1:61677)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.632848755Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"ds_connection connected from Ok(127.0.0.1:54332)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.633038855Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"ds_connection connected from Ok(127.0.0.1:41854)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.633081361Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.633630343Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"1","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.63365895Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"2","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.633674369Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"0","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.634308303Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"0","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.634372356Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"2","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.634405312Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"1","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.634931108Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635046286Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[0]R flush_numbers[0..12]: [325, 325, 333, 329, 326, 322, 330, 332, 332, 329, 311, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635117048Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[0]R generation[0..12]: [190, 190, 195, 193, 190, 188, 194, 194, 194, 193, 181, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635138366Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635153365Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[1]R flush_numbers[0..12]: [325, 325, 333, 329, 326, 322, 330, 332, 332, 329, 311, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635167813Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[1]R generation[0..12]: [190, 190, 195, 193, 190, 188, 194, 194, 194, 193, 181, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635181732Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.6352008Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[2]R flush_numbers[0..12]: [325, 325, 333, 329, 326, 322, 330, 332, 332, 329, 311, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635215328Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[2]R generation[0..12]: [190, 190, 195, 193, 190, 188, 194, 194, 194, 193, 181, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635229227Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635243075Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635255814Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635268853Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635281541Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.63529429Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"":"downstairs","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635307598Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635319537Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"0","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635333046Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"1","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635346494Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"client":"2","":"downstairs","session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635368582Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635380881Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"20b05080-0ad4-400f-a643-8280f016a4ff is now active with session: 58e31f0e-494a-460d-85aa-bab8befa6e28","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.63539378Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635408578Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976,"session_id":"58e31f0e-494a-460d-85aa-bab8befa6e28"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635421027Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} {"msg":"Activated sub_volume 20b05080-0ad4-400f-a643-8280f016a4ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:03.635432285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 4096 data: 14 02/30 Flush 03/30 Write block 149 len 40960 data: 14 12 9 14 14 14 13 12 16 19 04/30 Read block 324 len 28672 05/30 Read block 87 len 8192 06/30 Read block 480 len 16384 07/30 Write block 543 len 20480 data: 11 11 16 17 17 08/30 Write block 487 len 24576 data: 12 13 12 12 10 12 09/30 Write block 218 len 28672 data: 16 16 12 10 14 10 12 10/30 Write block 106 len 12288 data: 12 14 15 11/30 Read block 409 len 40960 12/30 Read block 249 len 40960 13/30 Read block 339 len 12288 14/30 Read block 31 len 40960 15/30 Write block 259 len 12288 data: 17 15 14 16/30 Write block 584 len 40960 data: 17 18 15 16 11 14 15 12 10 13 17/30 Write block 272 len 36864 data: 15 14 15 13 12 12 11 14 16 18/30 Write block 372 len 32768 data: 13 10 10 14 12 14 14 15 19/30 Read block 484 len 24576 20/30 Read block 154 len 12288 21/30 Write block 240 len 20480 data: 13 17 18 18 18 22/30 Write block 506 len 4096 data: 15 23/30 Write block 426 len 12288 data: 17 20 17 24/30 Write block 579 len 40960 data: 17 11 13 13 15 18 19 16 17 12 25/30 Flush 26/30 Write block 163 len 12288 data: 14 13 11 27/30 Read block 327 len 36864 28/30 Read block 573 len 16384 29/30 Write block 463 len 16384 data: 13 15 12 11 30/30 Write block 489 len 24576 data: 13 13 11 13 11 10 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:03.655733548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1976} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 196 196 196 334 334 334 F F F 3 060-079 193 193 193 329 329 329 F F F 4 080-099 190 190 190 326 326 326 F F F 5 100-119 196 196 196 335 335 335 F F F 6 120-139 194 194 194 330 330 330 F F F 7 140-159 196 196 196 335 335 335 F F F 8 160-179 194 194 194 332 332 332 T T T 9 180-199 193 193 193 329 329 329 F F F 10 200-219 196 196 196 335 335 335 F F F 11 220-239 196 196 196 335 335 335 F F F 12 240-259 196 196 196 335 335 335 F F F 13 260-279 196 196 196 335 335 335 F F F 14 280-299 196 196 196 335 335 335 F F F 15 300-319 194 194 194 332 332 332 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 194 194 194 332 332 332 F F F 18 360-379 196 196 196 335 335 335 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 194 194 194 332 332 332 F F F 21 420-439 196 196 196 335 335 335 F F F 22 440-459 195 195 195 333 333 333 F F F 23 460-479 194 194 194 332 332 332 T T T 24 480-499 196 196 196 335 335 335 T T T 25 500-519 196 196 196 335 335 335 F F F 26 520-539 193 193 193 329 329 329 F F F 27 540-559 196 196 196 335 335 335 F F F 28 560-579 196 196 196 335 335 335 F F F 29 580-599 196 196 196 335 335 335 F F F Max gen: 196, Max flush: 335 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 196 196 196 334 334 334 F F F 3 060-079 193 193 193 329 329 329 F F F 4 080-099 190 190 190 326 326 326 F F F 5 100-119 196 196 196 335 335 335 F F F 6 120-139 194 194 194 330 330 330 F F F 7 140-159 196 196 196 335 335 335 F F F 8 160-179 197 197 197 336 336 336 F F F 9 180-199 193 193 193 329 329 329 F F F 10 200-219 196 196 196 335 335 335 F F F 11 220-239 196 196 196 335 335 335 F F F 12 240-259 196 196 196 335 335 335 F F F 13 260-279 196 196 196 335 335 335 F F F 14 280-299 196 196 196 335 335 335 F F F 15 300-319 194 194 194 332 332 332 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 194 194 194 332 332 332 F F F 18 360-379 196 196 196 335 335 335 F F F 19 380-399 190 190 190 324 324 324 F F F 20 400-419 194 194 194 332 332 332 F F F 21 420-439 196 196 196 335 335 335 F F F 22 440-459 195 195 195 333 333 333 F F F 23 460-479 197 197 197 336 336 336 F F F 24 480-499 197 197 197 336 336 336 F F F 25 500-519 196 196 196 335 335 335 F F F 26 520-539 193 193 193 329 329 329 F F F 27 540-559 196 196 196 335 335 335 F F F 28 560-579 196 196 196 335 335 335 F F F 29 580-599 196 196 196 335 335 335 F F F Max gen: 197, Max flush: 336 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.883436785Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.927577762Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.928160831Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7ce02df48de2b1dbc251235166766a7d2d332248\",\n git_commit_timestamp: \"2025-03-18T04:42:36.000000000Z\",\n git_branch: \"renovate/bytes-1.x-lockfile\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.928203147Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.928225285Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.928243513Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} {"msg":"Crucible d4e61b76-58ce-4416-a581-5b8a094eeb89 has session id: 8a2804e0-8961-48e7-8e6c-571cc5dee9f9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.928787856Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Upstairs opts: Upstairs UUID: d4e61b76-58ce-4416-a581-5b8a094eeb89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.928831562Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Crucible stats registered with UUID: d4e61b76-58ce-4416-a581-5b8a094eeb89","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.928856789Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:07.928877887Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929373425Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929403432Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} {"msg":"d4e61b76-58ce-4416-a581-5b8a094eeb89 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929444778Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929460806Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"0","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929488353Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"0","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929502052Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"1","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.92951624Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"1","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929533348Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"2","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929560626Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"2","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929591272Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929616909Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929639977Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60118)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929666704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61623)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929816229Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42772)","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.929856704Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.930294839Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"1","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.930326666Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"0","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.93104802Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"0","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.931080357Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"1","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.931284936Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"2","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.931844807Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.931868075Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"2","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.931889383Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[0]R flush_numbers[0..12]: [325, 325, 334, 329, 326, 335, 330, 335, 336, 329, 335, 335]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932419077Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[0]R generation[0..12]: [190, 190, 196, 193, 190, 196, 194, 196, 197, 193, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932444845Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932459633Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[1]R flush_numbers[0..12]: [325, 325, 334, 329, 326, 335, 330, 335, 336, 329, 335, 335]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932473332Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[1]R generation[0..12]: [190, 190, 196, 193, 190, 196, 194, 196, 197, 193, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.93248691Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932500779Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[2]R flush_numbers[0..12]: [325, 325, 334, 329, 326, 335, 330, 335, 336, 329, 335, 335]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932514657Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[2]R generation[0..12]: [190, 190, 196, 193, 190, 196, 194, 196, 197, 193, 196, 196]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932528166Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932542315Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932555523Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932569702Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.93258251Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932602548Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"":"downstairs","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932615617Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932630745Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"0","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932644194Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"1","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932657212Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"client":"2","":"downstairs","session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932670351Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.93268273Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"d4e61b76-58ce-4416-a581-5b8a094eeb89 is now active with session: 8a2804e0-8961-48e7-8e6c-571cc5dee9f9","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932695568Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932707527Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984,"session_id":"8a2804e0-8961-48e7-8e6c-571cc5dee9f9"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932719506Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} {"msg":"Activated sub_volume d4e61b76-58ce-4416-a581-5b8a094eeb89","v":0,"name":"crucible","level":30,"time":"2025-03-18T05:09:07.932730555Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 57 len 40960 data: 17 21 25 22 19 18 15 12 14 10 02/30 Write block 76 len 24576 data: 19 20 19 21 20 19 03/30 Read block 208 len 20480 04/30 Write block 382 len 12288 data: 22 24 21 05/30 Write block 161 len 12288 data: 15 13 15 06/30 Read block 322 len 32768 07/30 Read block 10 len 20480 08/30 Read block 510 len 12288 09/30 Read block 435 len 24576 10/30 Read block 170 len 16384 11/30 Read block 482 len 20480 12/30 Read block 374 len 32768 13/30 Read block 575 len 40960 14/30 Read block 33 len 4096 15/30 Write block 538 len 4096 data: 12 16/30 Write block 256 len 8192 data: 13 16 17/30 Read block 187 len 12288 18/30 Write block 257 len 16384 data: 17 16 18 16 19/30 Write block 113 len 12288 data: 14 13 19 20/30 Read block 56 len 8192 21/30 Read block 261 len 28672 22/30 Read block 311 len 16384 23/30 Write block 454 len 28672 data: 15 16 20 17 14 13 13 24/30 Read block 469 len 12288 25/30 Read block 573 len 20480 26/30 Write block 385 len 4096 data: 16 27/30 Write block 297 len 12288 data: 18 17 19 28/30 Write block 164 len 4096 data: 14 29/30 Read block 196 len 40960 30/30 Read block 424 len 36864 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T05:09:07.952255669Z","hostname":"w-01JPKSTV77HZ4RY1P68MNYE4MB","pid":1984} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 196 196 196 334 334 334 T T T 3 060-079 193 193 193 329 329 329 T T T 4 080-099 190 190 190 326 326 326 T T T 5 100-119 196 196 196 335 335 335 T T T 6 120-139 194 194 194 330 330 330 F F F 7 140-159 196 196 196 335 335 335 F F F 8 160-179 197 197 197 336 336 336 T T T 9 180-199 193 193 193 329 329 329 F F F 10 200-219 196 196 196 335 335 335 F F F 11 220-239 196 196 196 335 335 335 F F F 12 240-259 196 196 196 335 335 335 T T T 13 260-279 196 196 196 335 335 335 T T T 14 280-299 196 196 196 335 335 335 T T T 15 300-319 194 194 194 332 332 332 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 194 194 194 332 332 332 F F F 18 360-379 196 196 196 335 335 335 F F F 19 380-399 190 190 190 324 324 324 T T T 20 400-419 194 194 194 332 332 332 F F F 21 420-439 196 196 196 335 335 335 F F F 22 440-459 195 195 195 333 333 333 T T T 23 460-479 197 197 197 336 336 336 T T T 24 480-499 197 197 197 336 336 336 F F F 25 500-519 196 196 196 335 335 335 F F F 26 520-539 193 193 193 329 329 329 T T T 27 540-559 196 196 196 335 335 335 F F F 28 560-579 196 196 196 335 335 335 F F F 29 580-599 196 196 196 335 335 335 F F F Max gen: 197, Max flush: 336 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 325 325 325 F F F 1 020-039 190 190 190 325 325 325 F F F 2 040-059 198 198 198 337 337 337 F F F 3 060-079 198 198 198 337 337 337 F F F 4 080-099 198 198 198 337 337 337 F F F 5 100-119 198 198 198 337 337 337 F F F 6 120-139 194 194 194 330 330 330 F F F 7 140-159 196 196 196 335 335 335 F F F 8 160-179 198 198 198 337 337 337 F F F 9 180-199 193 193 193 329 329 329 F F F 10 200-219 196 196 196 335 335 335 F F F 11 220-239 196 196 196 335 335 335 F F F 12 240-259 198 198 198 337 337 337 F F F 13 260-279 198 198 198 337 337 337 F F F 14 280-299 198 198 198 337 337 337 F F F 15 300-319 194 194 194 332 332 332 F F F 16 320-339 190 190 190 326 326 326 F F F 17 340-359 194 194 194 332 332 332 F F F 18 360-379 196 196 196 335 335 335 F F F 19 380-399 198 198 198 337 337 337 F F F 20 400-419 194 194 194 332 332 332 F F F 21 420-439 196 196 196 335 335 335 F F F 22 440-459 198 198 198 337 337 337 F F F 23 460-479 198 198 198 337 337 337 F F F 24 480-499 197 197 197 336 336 336 F F F 25 500-519 196 196 196 335 335 335 F F F 26 520-539 198 198 198 337 337 337 F F F 27 540-559 196 196 196 335 335 335 F F F 28 560-579 196 196 196 335 335 335 F F F 29 580-599 196 196 196 335 335 335 F F F Max gen: 198, Max flush: 337 10:15 Test duration Test completed real 10:14.506440257 user 49.732512708 sys 19.209704262 trap 0.066976206 tflt 0.087132789 dflt 0.123270655 kflt 0.006293725 lock 16:07:10.173296338 slp 49:46.617727748 lat 5.726337802 stop 0.062964473